effb3-512.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "metallic-evening",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-05-09T20:58:48.818115Z",
     "iopub.status.busy": "2021-05-09T20:58:48.817422Z",
     "iopub.status.idle": "2021-05-09T20:59:10.829039Z",
     "shell.execute_reply": "2021-05-09T20:59:10.828214Z"
    },
    "papermill": {
     "duration": 22.047876,
     "end_time": "2021-05-09T20:59:10.829215",
     "exception": false,
     "start_time": "2021-05-09T20:58:48.781339",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --quiet ../input/keras-efficientnet-whl/Keras_Applications-1.0.8-py3-none-any.whl\n",
    "!pip install --quiet ../input/keras-efficientnet-whl/efficientnet-1.1.1-py3-none-any.whl\n",
    "\n",
    "import os\n",
    "import math\n",
    "import re\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "import numpy as np\n",
    "import tensorflow.keras.backend as K\n",
    "import efficientnet.keras as efn\n",
    "import efficientnet\n",
    "import itertools\n",
    "import matplotlib\n",
    "import scipy\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "from matplotlib import pyplot as plt\n",
    "from datetime import datetime\n",
    "from functools import partial\n",
    "from kaggle_datasets import KaggleDatasets\n",
    "import pickle\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# import cudf\n",
    "# import cuml\n",
    "# import cupy\n",
    "# from cuml.feature_extraction.text import TfidfVectorizer\n",
    "# from cuml.neighbors import NearestNeighbors\n",
    "\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "SEED = 42\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "tf.random.set_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "laden-coach",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T20:59:10.872168Z",
     "iopub.status.busy": "2021-05-09T20:59:10.871405Z",
     "iopub.status.idle": "2021-05-09T21:00:09.542555Z",
     "shell.execute_reply": "2021-05-09T21:00:09.543081Z"
    },
    "papermill": {
     "duration": 58.701454,
     "end_time": "2021-05-09T21:00:09.543261",
     "exception": false,
     "start_time": "2021-05-09T20:59:10.841807",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device:  grpc://10.0.0.2:8470\n",
      "TPU\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['gs://kds-76d5d18990c3e2732726b4677287b4f3d5ac09139df2b3f090c78094/train.tfrec']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try:\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Device: ', tpu.master())\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "    DEVICE = \"TPU\"\n",
    "except:\n",
    "    DEVICE = 'notTPU'\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "print(DEVICE)\n",
    "\n",
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "GCS_PATH = KaggleDatasets().get_gcs_path(f'shopee-tfrecord-512-all-010521')\n",
    "REPLICAS = strategy.num_replicas_in_sync\n",
    "BATCH_SIZE = 32 * REPLICAS\n",
    "FILENAMES = tf.io.gfile.glob(GCS_PATH + f'/*.tfrec')\n",
    "FILENAMES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "utility-yahoo",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.574420Z",
     "iopub.status.busy": "2021-05-09T21:00:09.573804Z",
     "iopub.status.idle": "2021-05-09T21:00:09.575855Z",
     "shell.execute_reply": "2021-05-09T21:00:09.576346Z"
    },
    "papermill": {
     "duration": 0.020523,
     "end_time": "2021-05-09T21:00:09.576513",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.555990",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "IMAGE_SIZE = (512, 512)\n",
    "\n",
    "EPOCHS = 100\n",
    "LR = 0.001\n",
    "\n",
    "CLASSES = 11014"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "moral-nomination",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.605881Z",
     "iopub.status.busy": "2021-05-09T21:00:09.605240Z",
     "iopub.status.idle": "2021-05-09T21:00:09.618674Z",
     "shell.execute_reply": "2021-05-09T21:00:09.619195Z"
    },
    "papermill": {
     "duration": 0.029617,
     "end_time": "2021-05-09T21:00:09.619363",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.589746",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def decode_image(image, img_size):\n",
    "    image = tf.image.decode_jpeg(image, channels=3)\n",
    "    image = tf.image.resize(image, img_size)\n",
    "    image = tf.cast(image, tf.float32) / 255.0\n",
    "    return tf.reshape(image, [*img_size, 3])\n",
    "\n",
    "# parse tfrecord files\n",
    "def _parse_image(proto, train):\n",
    "    features = {\n",
    "        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n",
    "        \"title_vec\": tf.io.FixedLenFeature([100], tf.float32),  # shape [] means single element\n",
    "        \"phash_vec\": tf.io.FixedLenFeature([25], tf.float32),\n",
    "        \"label\": tf.io.FixedLenFeature([], tf.int64),  \n",
    "    }\n",
    "    return tf.io.parse_single_example(proto, features)\n",
    "\n",
    "# get image and label_group\n",
    "def get_image_and_label(proto, train, img_size):\n",
    "    sample = _parse_image(proto, train=train)\n",
    "    sample[\"image\"] = decode_image(sample[\"image\"], img_size)\n",
    "    image = sample[\"image\"]\n",
    "    title_vec = sample[\"title_vec\"]\n",
    "    phash_vec = sample[\"phash_vec\"]\n",
    "    label = tf.cast(sample['label'], tf.int32)\n",
    "#     label = tf.one_hot(tf.cast(sample['label'], tf.int32), len(CLASSES))\n",
    "    return (image, title_vec, label), label\n",
    "\n",
    "def get_training_dataset(tfr, batchsize, img_size = (512,512)):\n",
    "    return tfr.map(partial(get_image_and_label, train=True, img_size=img_size)).repeat().map(\n",
    "        lambda x,y: image_augmentation(x,y)).shuffle(1000).batch(batchsize).prefetch(AUTO)\n",
    "\n",
    "def get_validation_dataset(tfr, batchsize, img_size=(512, 512), train=True):\n",
    "    return tfr.map(partial(get_image_and_label, train=train, img_size=img_size)).batch(batchsize).prefetch(AUTO)\n",
    "\n",
    "def get_tfrecord_size(tfrecord):\n",
    "    return sum(1 for _ in tfrecord)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "peripheral-shift",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.653496Z",
     "iopub.status.busy": "2021-05-09T21:00:09.652491Z",
     "iopub.status.idle": "2021-05-09T21:00:09.654755Z",
     "shell.execute_reply": "2021-05-09T21:00:09.655223Z"
    },
    "papermill": {
     "duration": 0.022784,
     "end_time": "2021-05-09T21:00:09.655389",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.632605",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Augmentation function & tf.image functions (flip, brightness, contrast, saturation, hue)\n",
    "# for train\n",
    "# @tf.function\n",
    "def image_augmentation(iw, label):\n",
    "    image, title_vec, label = iw\n",
    "\n",
    "    image = tf.image.random_flip_left_right(image)\n",
    "    image = tf.image.random_flip_up_down(image)\n",
    "    image = tf.image.random_hue(image, 0.01)\n",
    "    image = tf.image.random_saturation(image, 0.70, 1.30)\n",
    "    image = tf.image.random_contrast(image, 0.80, 1.20)\n",
    "    image = tf.image.random_brightness(image, 0.10)\n",
    "\n",
    "    return ((image, title_vec, label), label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "announced-stockholm",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.685524Z",
     "iopub.status.busy": "2021-05-09T21:00:09.684555Z",
     "iopub.status.idle": "2021-05-09T21:00:09.689978Z",
     "shell.execute_reply": "2021-05-09T21:00:09.690423Z"
    },
    "papermill": {
     "duration": 0.022002,
     "end_time": "2021-05-09T21:00:09.690619",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.668617",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model, Sequential\n",
    "import keras.backend as K\n",
    "from keras.optimizers import SGD\n",
    "from tensorflow.keras.layers import (\n",
    "    Input, \n",
    "    Flatten, \n",
    "    Dense, \n",
    "    Dropout, \n",
    "    AveragePooling2D, \n",
    "    GlobalAveragePooling2D, \n",
    "    SpatialDropout2D, \n",
    "    BatchNormalization, \n",
    "    Activation, \n",
    "    Concatenate,\n",
    "    Embedding,\n",
    "    GlobalAveragePooling1D,\n",
    "    Lambda\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "varied-transparency",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.721271Z",
     "iopub.status.busy": "2021-05-09T21:00:09.720270Z",
     "iopub.status.idle": "2021-05-09T21:00:09.733133Z",
     "shell.execute_reply": "2021-05-09T21:00:09.732454Z"
    },
    "papermill": {
     "duration": 0.029333,
     "end_time": "2021-05-09T21:00:09.733273",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.703940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plt_lr(epoch_count):\n",
    "    if epoch_count > 50:\n",
    "        epoch_count = 50\n",
    "    \n",
    "    rng = [i for i in range(epoch_count)]\n",
    "\n",
    "    plt.figure()\n",
    "    y = [lrfn(x) for x in rng]\n",
    "    plt.title(f'Learning rate schedule: {y[0]} to {y[epoch_count-1]}')\n",
    "    plt.plot(rng, y)\n",
    "\n",
    "def plt_acc(h):\n",
    "    plt.figure()\n",
    "    plt.plot(h.history[\"sparse_categorical_accuracy\"])\n",
    "    if 'val_sparse_categorical_accuracy' in h.history:\n",
    "        plt.plot(h.history[\"val_sparse_categorical_accuracy\"]) \n",
    "        plt.legend([\"training\",\"validation\"])       \n",
    "    else:\n",
    "        plt.legend([\"training\"])\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.title(\"Sparse Categorical Accuracy\")\n",
    "    plt.show()\n",
    "    \n",
    "def plt_f1_score(h):\n",
    "    plt.figure()\n",
    "    plt.plot(h.history[\"f1_score\"])\n",
    "    if 'f1_score' in h.history:\n",
    "        plt.plot(h.history[\"f1_score\"]) \n",
    "        plt.legend([\"training\",\"validation\"])       \n",
    "    else:\n",
    "        plt.legend([\"training\"])\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.title(\"F1 Score\")\n",
    "    plt.show()\n",
    "\n",
    "def plt_loss(h):\n",
    "    plt.figure()\n",
    "    plt.plot(h.history[\"loss\"])\n",
    "    if 'val_loss' in h.history:\n",
    "        plt.plot(h.history[\"val_loss\"]) \n",
    "        plt.legend([\"training\",\"validation\"])       \n",
    "    else:\n",
    "        plt.legend([\"training\"])\n",
    "    plt.legend([\"training\",\"validation\"])\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.title(\"Loss\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "continued-poultry",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.767815Z",
     "iopub.status.busy": "2021-05-09T21:00:09.766782Z",
     "iopub.status.idle": "2021-05-09T21:00:09.770106Z",
     "shell.execute_reply": "2021-05-09T21:00:09.769428Z"
    },
    "papermill": {
     "duration": 0.02348,
     "end_time": "2021-05-09T21:00:09.770247",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.746767",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "es_val_f1 = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='f1_score', min_delta=0.001, patience=10, verbose=1, mode='max',\n",
    "    baseline=None, restore_best_weights=True\n",
    ")\n",
    "\n",
    "es_val_f1_stage2 = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='f1_score', min_delta=0.001, patience=15, verbose=1, mode='max',\n",
    "    baseline=None, restore_best_weights=True\n",
    ")\n",
    "\n",
    "cb_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "    \"backup.h5\",\n",
    "    monitor=\"f1_score\", verbose=1, mode='max', save_best_only=True)\n",
    "cb_checkpoint._supports_tf_logs = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "higher-drive",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.801139Z",
     "iopub.status.busy": "2021-05-09T21:00:09.800479Z",
     "iopub.status.idle": "2021-05-09T21:00:09.814948Z",
     "shell.execute_reply": "2021-05-09T21:00:09.814373Z"
    },
    "papermill": {
     "duration": 0.031031,
     "end_time": "2021-05-09T21:00:09.815120",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.784089",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# swish activation\n",
    "from keras.backend import sigmoid\n",
    "def swish(x, beta = 1):\n",
    "    return (x * sigmoid(beta * x))\n",
    "\n",
    "from keras.utils.generic_utils import get_custom_objects\n",
    "from keras.layers import Activation\n",
    "\n",
    "get_custom_objects().update({'swish': Activation(swish)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "previous-alert",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.848501Z",
     "iopub.status.busy": "2021-05-09T21:00:09.847849Z",
     "iopub.status.idle": "2021-05-09T21:00:09.850380Z",
     "shell.execute_reply": "2021-05-09T21:00:09.849760Z"
    },
    "papermill": {
     "duration": 0.021561,
     "end_time": "2021-05-09T21:00:09.850516",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.828955",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, multiply, Reshape\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D,Bidirectional, Conv1D, GlobalMaxPooling1D, Conv2D\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "oriental-apache",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.882248Z",
     "iopub.status.busy": "2021-05-09T21:00:09.881419Z",
     "iopub.status.idle": "2021-05-09T21:00:09.897976Z",
     "shell.execute_reply": "2021-05-09T21:00:09.898494Z"
    },
    "papermill": {
     "duration": 0.033912,
     "end_time": "2021-05-09T21:00:09.898705",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.864793",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ArcMarginProduct(keras.layers.Layer):\n",
    "    '''\n",
    "    Implements large margin arc distance.\n",
    "\n",
    "    Reference:\n",
    "        https://arxiv.org/pdf/1801.07698.pdf\n",
    "        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n",
    "            blob/master/src/modeling/metric_learning.py\n",
    "    '''\n",
    "    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n",
    "                 ls_eps=0.0, **kwargs):\n",
    "\n",
    "        super(ArcMarginProduct, self).__init__(**kwargs)\n",
    "\n",
    "        self.n_classes = n_classes\n",
    "        self.s = s\n",
    "        self.m = m\n",
    "        self.ls_eps = ls_eps\n",
    "        self.easy_margin = easy_margin\n",
    "        self.cos_m = tf.math.cos(m)\n",
    "        self.sin_m = tf.math.sin(m)\n",
    "        self.th = tf.math.cos(math.pi - m)\n",
    "        self.mm = tf.math.sin(math.pi - m) * m\n",
    "\n",
    "    def get_config(self):\n",
    "\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'n_classes': self.n_classes,\n",
    "            's': self.s,\n",
    "            'm': self.m,\n",
    "            'ls_eps': self.ls_eps,\n",
    "            'easy_margin': self.easy_margin,\n",
    "        })\n",
    "        return config\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        super(ArcMarginProduct, self).build(input_shape[0])\n",
    "\n",
    "        self.W = self.add_weight(\n",
    "            name='W',\n",
    "            shape=(int(input_shape[0][-1]), self.n_classes),\n",
    "            initializer='glorot_uniform',\n",
    "            dtype='float32',\n",
    "            trainable=True,\n",
    "            regularizer=None)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        X, y = inputs\n",
    "        y = tf.cast(y, dtype=tf.int32)\n",
    "        cosine = tf.matmul(\n",
    "            tf.math.l2_normalize(X, axis=1),\n",
    "            tf.math.l2_normalize(self.W, axis=0)\n",
    "        )\n",
    "        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n",
    "        phi = cosine * self.cos_m - sine * self.sin_m\n",
    "        if self.easy_margin:\n",
    "            phi = tf.where(cosine > 0, phi, cosine)\n",
    "        else:\n",
    "            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n",
    "        one_hot = tf.cast(\n",
    "            tf.one_hot(y, depth=self.n_classes),\n",
    "            dtype=cosine.dtype\n",
    "        )\n",
    "        if self.ls_eps > 0:\n",
    "            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n",
    "\n",
    "        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n",
    "        output *= self.s\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "employed-worry",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.930357Z",
     "iopub.status.busy": "2021-05-09T21:00:09.929640Z",
     "iopub.status.idle": "2021-05-09T21:00:09.946564Z",
     "shell.execute_reply": "2021-05-09T21:00:09.947099Z"
    },
    "papermill": {
     "duration": 0.034417,
     "end_time": "2021-05-09T21:00:09.947283",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.912866",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def effb3(weights='noisy-student'):\n",
    "    img_inp = Input(shape=(*IMAGE_SIZE, 3))\n",
    "    effb3 = efn.EfficientNetB3(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    pt_depth = effb3.layers[-1].get_output_shape_at(0)[-1]\n",
    "    pt_features = effb3(img_inp)\n",
    "    bn_features = BatchNormalization()(pt_features)\n",
    "    \n",
    "    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n",
    "    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid')(attn_layer)\n",
    "    \n",
    "    up_c2_w = np.ones((1, 1, 1, pt_depth))\n",
    "    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', activation = 'linear', \n",
    "                   use_bias = False, weights = [up_c2_w])\n",
    "    up_c2.trainable = False\n",
    "    attn_layer = up_c2(attn_layer)\n",
    "    \n",
    "    mask_features = multiply([attn_layer, bn_features])\n",
    "    gap_features = GlobalAveragePooling2D()(mask_features)\n",
    "    gap_mask = GlobalAveragePooling2D()(attn_layer)\n",
    "    # to account for missing values from the attention model\n",
    "    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n",
    "    gap_dr = Dropout(0.25)(gap)\n",
    "    img_embed = Dense(2400, activation = 'swish', name='img-embed')(gap_dr)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([img_embed, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[img_inp, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "# model = effb5()\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "august-infrared",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:09.979201Z",
     "iopub.status.busy": "2021-05-09T21:00:09.978529Z",
     "iopub.status.idle": "2021-05-09T21:00:09.985135Z",
     "shell.execute_reply": "2021-05-09T21:00:09.985724Z"
    },
    "papermill": {
     "duration": 0.024285,
     "end_time": "2021-05-09T21:00:09.985927",
     "exception": false,
     "start_time": "2021-05-09T21:00:09.961642",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_lr_callback():\n",
    "    lr_start   = 0.000001\n",
    "    lr_max     = 0.000005 * BATCH_SIZE\n",
    "    lr_min     = 0.000001\n",
    "    lr_ramp_ep = 5\n",
    "    lr_sus_ep  = 0\n",
    "    lr_decay   = 0.8\n",
    "   \n",
    "    def lrfn(epoch):\n",
    "        if epoch < lr_ramp_ep:\n",
    "            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n",
    "        elif epoch < lr_ramp_ep + lr_sus_ep:\n",
    "            lr = lr_max    \n",
    "        else:\n",
    "            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n",
    "        return lr\n",
    "\n",
    "    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n",
    "    return lr_callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dedicated-nothing",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:00:10.018691Z",
     "iopub.status.busy": "2021-05-09T21:00:10.017988Z",
     "iopub.status.idle": "2021-05-09T21:01:10.476299Z",
     "shell.execute_reply": "2021-05-09T21:01:10.476885Z"
    },
    "papermill": {
     "duration": 60.476125,
     "end_time": "2021-05-09T21:01:10.477115",
     "exception": false,
     "start_time": "2021-05-09T21:00:10.000990",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "\n",
    "# for randomness\n",
    "ignore_order = tf.data.Options()\n",
    "ignore_order.experimental_deterministic = False\n",
    "\n",
    "train_records = tf.data.TFRecordDataset(FILENAMES, num_parallel_reads=AUTO)\n",
    "train_records = train_records.with_options(ignore_order)\n",
    "train_data = get_training_dataset(train_records, BATCH_SIZE, IMAGE_SIZE)\n",
    "train_data = train_data.map(lambda x, y: (x, y))    \n",
    "train_size = get_tfrecord_size(train_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "corresponding-teaching",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-09T21:01:10.522866Z",
     "iopub.status.busy": "2021-05-09T21:01:10.518619Z",
     "iopub.status.idle": "2021-05-10T01:58:13.356740Z",
     "shell.execute_reply": "2021-05-10T01:58:13.355536Z"
    },
    "papermill": {
     "duration": 17822.865038,
     "end_time": "2021-05-10T01:58:13.356965",
     "exception": false,
     "start_time": "2021-05-09T21:01:10.491927",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b3_noisy-student_notop.h5\n",
      "43933696/43933088 [==============================] - 1s 0us/step\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 512, 512, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "efficientnet-b3 (Functional)    (None, 16, 16, 1536) 10783528    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 16, 16, 1536) 6144        efficientnet-b3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 16, 16, 1536) 0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "title-input (InputLayer)        [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 16, 16, 64)   98368       dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "title-embed (Embedding)         (None, 100, 150)     3750000     title-input[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 16, 16, 16)   1040        conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 100, 150)     0           title-embed[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 16, 16, 8)    136         conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d (Conv1D)                 (None, 98, 300)      135300      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 16, 16, 1)    9           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 300)          0           conv1d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 16, 16, 1536) 1536        conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "title-dense1 (Dense)            (None, 720)          216720      global_max_pooling1d[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "multiply (Multiply)             (None, 16, 16, 1536) 0           conv2d_4[0][0]                   \n",
      "                                                                 batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "title-act1 (Activation)         (None, 720)          0           title-dense1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d (Globa (None, 1536)         0           multiply[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_1 (Glo (None, 1536)         0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 720)          0           title-act1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "RescaleGAP (Lambda)             (None, 1536)         0           global_average_pooling2d[0][0]   \n",
      "                                                                 global_average_pooling2d_1[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "title-dense2 (Dense)            (None, 650)          468650      dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1536)         0           RescaleGAP[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "title-bn2 (BatchNormalization)  (None, 650)          2600        title-dense2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "img-embed (Dense)               (None, 2400)         3688800     dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "title-act2 (Activation)         (None, 650)          0           title-bn2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 3050)         0           img-embed[0][0]                  \n",
      "                                                                 title-act2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "arc-input (InputLayer)          [(None,)]            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "head/arc_margin (ArcMarginProdu (None, 11014)        33592700    concatenate[0][0]                \n",
      "                                                                 arc-input[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 11014)        121319210   head/arc_margin[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 174,064,741\n",
      "Trainable params: 173,971,537\n",
      "Non-trainable params: 93,204\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/38\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 1e-06.\n",
      "133/133 [==============================] - 592s 3s/step - loss: 9.4703 - sparse_categorical_accuracy: 5.7990e-05\n",
      "Epoch 2/38\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.00025680000000000006.\n",
      "133/133 [==============================] - 468s 4s/step - loss: 9.3342 - sparse_categorical_accuracy: 0.0049\n",
      "Epoch 3/38\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0005126000000000001.\n",
      "133/133 [==============================] - 468s 4s/step - loss: 8.1518 - sparse_categorical_accuracy: 0.0802\n",
      "Epoch 4/38\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.0007684000000000001.\n",
      "133/133 [==============================] - 468s 4s/step - loss: 5.9216 - sparse_categorical_accuracy: 0.2230\n",
      "Epoch 5/38\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.0010242.\n",
      "133/133 [==============================] - 467s 4s/step - loss: 3.4697 - sparse_categorical_accuracy: 0.4841\n",
      "Epoch 6/38\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.00128.\n",
      "133/133 [==============================] - 467s 4s/step - loss: 1.4696 - sparse_categorical_accuracy: 0.7894\n",
      "Epoch 7/38\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.0010242.\n",
      "133/133 [==============================] - 466s 4s/step - loss: 0.4407 - sparse_categorical_accuracy: 0.9268\n",
      "Epoch 8/38\n",
      "\n",
      "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0008195600000000003.\n",
      "133/133 [==============================] - 466s 4s/step - loss: 0.1969 - sparse_categorical_accuracy: 0.9635\n",
      "Epoch 9/38\n",
      "\n",
      "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0006558480000000003.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.1273 - sparse_categorical_accuracy: 0.9764\n",
      "Epoch 10/38\n",
      "\n",
      "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0005248784000000002.\n",
      "133/133 [==============================] - 461s 3s/step - loss: 0.0755 - sparse_categorical_accuracy: 0.9876\n",
      "Epoch 11/38\n",
      "\n",
      "Epoch 00011: LearningRateScheduler reducing learning rate to 0.0004201027200000002.\n",
      "133/133 [==============================] - 459s 3s/step - loss: 0.0521 - sparse_categorical_accuracy: 0.9918\n",
      "Epoch 12/38\n",
      "\n",
      "Epoch 00012: LearningRateScheduler reducing learning rate to 0.0003362821760000002.\n",
      "133/133 [==============================] - 465s 3s/step - loss: 0.0430 - sparse_categorical_accuracy: 0.9930\n",
      "Epoch 13/38\n",
      "\n",
      "Epoch 00013: LearningRateScheduler reducing learning rate to 0.00026922574080000017.\n",
      "133/133 [==============================] - 463s 3s/step - loss: 0.0326 - sparse_categorical_accuracy: 0.9952\n",
      "Epoch 14/38\n",
      "\n",
      "Epoch 00014: LearningRateScheduler reducing learning rate to 0.00021558059264000014.\n",
      "133/133 [==============================] - 462s 3s/step - loss: 0.0234 - sparse_categorical_accuracy: 0.9963\n",
      "Epoch 15/38\n",
      "\n",
      "Epoch 00015: LearningRateScheduler reducing learning rate to 0.0001726644741120001.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.0167 - sparse_categorical_accuracy: 0.9973\n",
      "Epoch 16/38\n",
      "\n",
      "Epoch 00016: LearningRateScheduler reducing learning rate to 0.0001383315792896001.\n",
      "133/133 [==============================] - 466s 4s/step - loss: 0.0132 - sparse_categorical_accuracy: 0.9978\n",
      "Epoch 17/38\n",
      "\n",
      "Epoch 00017: LearningRateScheduler reducing learning rate to 0.00011086526343168008.\n",
      "133/133 [==============================] - 463s 3s/step - loss: 0.0104 - sparse_categorical_accuracy: 0.9988\n",
      "Epoch 18/38\n",
      "\n",
      "Epoch 00018: LearningRateScheduler reducing learning rate to 8.889221074534406e-05.\n",
      "133/133 [==============================] - 461s 3s/step - loss: 0.0094 - sparse_categorical_accuracy: 0.9988\n",
      "Epoch 19/38\n",
      "\n",
      "Epoch 00019: LearningRateScheduler reducing learning rate to 7.131376859627525e-05.\n",
      "133/133 [==============================] - 465s 3s/step - loss: 0.0079 - sparse_categorical_accuracy: 0.9989\n",
      "Epoch 20/38\n",
      "\n",
      "Epoch 00020: LearningRateScheduler reducing learning rate to 5.725101487702021e-05.\n",
      "133/133 [==============================] - 465s 3s/step - loss: 0.0065 - sparse_categorical_accuracy: 0.9991\n",
      "Epoch 21/38\n",
      "\n",
      "Epoch 00021: LearningRateScheduler reducing learning rate to 4.600081190161617e-05.\n",
      "133/133 [==============================] - 465s 3s/step - loss: 0.0053 - sparse_categorical_accuracy: 0.9993\n",
      "Epoch 22/38\n",
      "\n",
      "Epoch 00022: LearningRateScheduler reducing learning rate to 3.7000649521292936e-05.\n",
      "133/133 [==============================] - 463s 3s/step - loss: 0.0048 - sparse_categorical_accuracy: 0.9992\n",
      "Epoch 23/38\n",
      "\n",
      "Epoch 00023: LearningRateScheduler reducing learning rate to 2.9800519617034356e-05.\n",
      "133/133 [==============================] - 466s 4s/step - loss: 0.0041 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 24/38\n",
      "\n",
      "Epoch 00024: LearningRateScheduler reducing learning rate to 2.4040415693627487e-05.\n",
      "133/133 [==============================] - 463s 3s/step - loss: 0.0038 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 25/38\n",
      "\n",
      "Epoch 00025: LearningRateScheduler reducing learning rate to 1.9432332554901988e-05.\n",
      "133/133 [==============================] - 462s 3s/step - loss: 0.0043 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 26/38\n",
      "\n",
      "Epoch 00026: LearningRateScheduler reducing learning rate to 1.574586604392159e-05.\n",
      "133/133 [==============================] - 465s 3s/step - loss: 0.0041 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 27/38\n",
      "\n",
      "Epoch 00027: LearningRateScheduler reducing learning rate to 1.2796692835137275e-05.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.0035 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 28/38\n",
      "\n",
      "Epoch 00028: LearningRateScheduler reducing learning rate to 1.043735426810982e-05.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.0032 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 29/38\n",
      "\n",
      "Epoch 00029: LearningRateScheduler reducing learning rate to 8.549883414487856e-06.\n",
      "133/133 [==============================] - 466s 4s/step - loss: 0.0031 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 30/38\n",
      "\n",
      "Epoch 00030: LearningRateScheduler reducing learning rate to 7.0399067315902855e-06.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.0032 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 31/38\n",
      "\n",
      "Epoch 00031: LearningRateScheduler reducing learning rate to 5.831925385272228e-06.\n",
      "133/133 [==============================] - 460s 3s/step - loss: 0.0031 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 32/38\n",
      "\n",
      "Epoch 00032: LearningRateScheduler reducing learning rate to 4.865540308217783e-06.\n",
      "133/133 [==============================] - 467s 4s/step - loss: 0.0034 - sparse_categorical_accuracy: 0.9994\n",
      "Epoch 33/38\n",
      "\n",
      "Epoch 00033: LearningRateScheduler reducing learning rate to 4.092432246574227e-06.\n",
      "133/133 [==============================] - 460s 3s/step - loss: 0.0033 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 34/38\n",
      "\n",
      "Epoch 00034: LearningRateScheduler reducing learning rate to 3.473945797259381e-06.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.0031 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 35/38\n",
      "\n",
      "Epoch 00035: LearningRateScheduler reducing learning rate to 2.979156637807505e-06.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.0028 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 36/38\n",
      "\n",
      "Epoch 00036: LearningRateScheduler reducing learning rate to 2.5833253102460042e-06.\n",
      "133/133 [==============================] - 465s 3s/step - loss: 0.0031 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 37/38\n",
      "\n",
      "Epoch 00037: LearningRateScheduler reducing learning rate to 2.2666602481968035e-06.\n",
      "133/133 [==============================] - 461s 3s/step - loss: 0.0030 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 38/38\n",
      "\n",
      "Epoch 00038: LearningRateScheduler reducing learning rate to 2.013328198557443e-06.\n",
      "133/133 [==============================] - 464s 3s/step - loss: 0.0028 - sparse_categorical_accuracy: 0.9996\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmY0lEQVR4nO3de3xdZZ3v8c8v917Spk1Cgd5bKrTWUqCUq07l4rTIAKMMt9EBXzrM4ciII86IHg8KZzxHB+ciIw6iw4gMiqCgpdaDMly0QAttKYVCgSa9pRe6k16TNmmS/Zs/1kq7s8llp9nJWnvn+3698srea6291m+vJN/95NnPfpa5OyIikvsKoi5ARESyQ4EuIpInFOgiInlCgS4ikicU6CIieUKBLiKSJxToIn1kZr8xs+v7uY8bzGxZtmoSAQV6zjOz883sBTPbZ2a7zex5Mzsz6rp6Y2bzzWypme0N637JzD6V4WOfNbPPDHSN3XH3Re7+wEAfx8xGmlmjmf1moI8l+UGBnsPMbBSwBPhXYCwwHrgDaBmAYxVlcV/nAE8DzwEnAZXATcCibB1jIFhgMP9mPk7ws7zYzI4fxONm9ectg8jd9ZWjX8A8YG8P628Ange+C+wD1gMXpqz/FPAmcACoBf4qZd0CoA74ErATeBCoIngB2QvsBv4AFITbnwj8AkgAG4HP9VDXMuCeHtaPCY+TAPaEtyeE674BtAPNQCPw3XD5KcDvwrreAq5K2V8l8ASwH3gZ+HtgWcr6c8Pl+8Lv56asezY85vPAIYIXoGeBz6Rs85cp5/EN4PRw+W1ATcryP0372Szr7hyE2zwdHns18MW0decDL4Q/i63ADeHyYcA/ApvD57MsXLYAqEvbxybgovD214GfA/8ZnqfPAPOBF8Nj7CD4PSpJefz7U875u8BXgOOBg0Blynanhz/L4qj/ZvL9K/IC9NWPHx6MAhqABwhat2PS1t8AtAF/AxQDV4d/5GPD9R8FpgMG/FH4h9gRRgvCx34LKA1D4f8B94b7KgY+GD62AFgF3A6UANMIXiD+uIuahxME8od7eF6VBK3T4UA58Cjwy5T16YE6Igy1TwFFwGlAPTArXP9w+DUcmBVuuyxcN5bgReOT4WOvDe9XphxrSxheReHzPnJ84M+AbcCZ4bk4CZicsu7E8PxcDTQBJ6T8bLoNdGAykAzrvRVYm7buQFhrcXi+5obr7gnrGw8UErxYlZJZoLcCV4T1DgPOAM4On/cUghetz4fblxOE/K1AWXj/rHDdUuCmlOP8M/CvUf+9DIWvyAvQVz9/gDAT+BFBa7oNWAyMC9fdAGwHLGX7l4BPdrOvXwK3hLcXAIeBspT1dwK/Ak5Ke9xZwJa0ZV8G/qOLY4wHHDilD89xLrAn5f6RQA3vXw38Ie0x3we+FoZaK3ByyrojLXSCIH8p7bEvcrTF+yxwZ9r6I8cHnuw4Zxk8jzXA5Sk/m54C/avAmpRz1g6clnJuH+/iMQUE/0Wc2sW6BfQe6L/vpf7PdxyX4MXklW62uxp4PrxdSPAf3vzB+psYyl/qQ89x7v6mu9/g7hOA2QQtwn9J2WSbh39Zoc3hNpjZIjNbHr4puRe4hKBbpUPC3ZtT7t8FbAB+a2a1ZnZbuHwycGL4BufecF9fAcZ1UfIegpbnCd09JzMbbmbfN7PNZrYf+D1QYWaF3TxkMnBW2vH/nODf/2qCFubWlO1Tb58YnpNUmwlCtKvt000k6Fbp6nn8hZmtSalpNp3Pb0/+AngIwN23Ebzf0DGyprtjVhG0lrusJwOdnqeZvc/MlpjZzvDn8H85Wn+3z5vgRX+WmU0FLgb2uftLx1iT9IECPY+4+3qC1vrslMXjzcxS7k8CtptZKUGf97cJWvQVBP8qp27baSpOdz/g7re6+zTgMuALZnYhQRBsdPeKlK9yd7+kixoPErSAP97DU7kVOJngX/hRwIfC5R21pU8RuhV4Lu34I939JoK+2zZgQsr2E1Nubyd4QUg1iaAbpcvz0MWxp6cvNLPJwA+Amwm6byqA1+l8frtkZucCM4Avh2G6k+C/oOvCNyu7PCZBN1NzN+uaCLqcOo5RSPBilyr9ef4bwfsuM8Kfw1dS6t9K0LX2HmEj4BHgEwT/AT3Y9TOVbFOg5zAzO8XMbjWzCeH9iQT/Ci9P2ew44HNmVmxmf0bQRbOUoK+7lDDwzGwR8JFejnepmZ0UvkDsI+gGSBJ04xwwsy+Z2TAzKzSz2T0Mn/w74AYz+1szqwz3faqZPRyuLyfoOthrZmMJuk5SvUvnMFkCvM/MPhk+z2IzO9PMZrp7O/AY8PWw5X8KQeu3w9LwsdeZWZGZXU3Qb72kp3OR4ofAF83sjHAUzElhmI8gCMhE+Pw+RecX2p5cT/Bm4yyC7qa54WOHEbxX8hBwkZldFdZcaWZz3T0J3A/8k5mdGP4czglfvN8Gyszso2ZWTNClU9pLHeUEb5A2huftppR1S4ATzOzzZlZqZuVmdlbK+h8TdCtdhgJ90CjQc9sBgpbbCjNrIgjy1wlauB1WELT26glGTFzp7g3ufgD4HEFLag9wHUH/e09mAE8RjC55Efieuz8ThualBMGzMTzWD4HRXe3E3V8ALgi/as1sN3AfQbhC0GU0LNzPcuD/p+3iO8CVZrbHzO4On8tHgGsIWtw7OfpmLgSt5NEcHa3zU8Khne7eENZ+K8EbzH8HXOru9b2ci47n8ijBef0Jwc/jlwRvOr9BMNrkRYIXoA8QjJTpkZmVAVcRvIm4M+VrY1j79e6+haB77FaCESZrgFPDXXwReI1gtM7u8DwUuPs+4H8S/Fy2EbTY63op54sEvxcHCP7b+FnK8z5A0J3yJwTn9R3gwynrnyd4sV/t7uldWjJArHP3quQTM7uB4M2786OuJU7M7FvA8e7er097Ss/M7GngJ+7+w6hrGSrUQpe8F3ZNzQm7ROYDnwYej7qufBZ2t51OSqteBp4+DSZDQTlBN8uJBN0f/0gwEkMGgJk9QDCe/Zawa0YGibpcRETyhLpcRETyRGRdLlVVVT5lypSoDi8ikpNWrVpV7+7pnyEAIgz0KVOmsHLlyqgOLyKSk8ys22Gg6nIREckTCnQRkTyhQBcRyRMKdBGRPKFAFxHJE70Gupndb2a7zOz1btabmd1tZhvMbK2ZnZ79MkVEpDeZtNB/BCzsYf0igln4ZgA3EsyhLCIig6zXceju/nszm9LDJpcDPw6virPczCrM7AR335GtIiW/uDvtSae13WlNJkkmnbakH/nennSSfnRZuzvJJCT96LrgiyP3vZvbyfAyi52O/556gn07HZdk5Mh+Or57+KjgNim3g/Wd9hsu8K6OET6+o6b0ffRUY2oNHecx9X6q1EuadL6+SefzkfrY3iYBeW+NXT/CwmtgmB29GobZ0Tq6qrurPaU+Nvh+9Hmk7yP1Z9JVfe/5GaWv6K+0c9zbVUwunHkccyZUZOfYKbLxwaLxdL50VV247D2BbmY3ErTimTRpUhYOLf3R1p7kYGs7B1vaaTrcxqHD7TS1tHVa1tzazqHD7TS3JjnU2n70flvwvaUtSVsySWubc7j96O3W9mRwvz243dqepC3ZcVvzB0l8WK/XkOrZsbwmVJeXxjbQM+bu9xFcyIB58+bpr3oANbe2U5No5N39zby7v4Wd+5rZdaCZnfuC+7sONFPfeLhP+ywpLKCsuIBhJYWUFRcyrLiQkqICSgoLKC4sYFRJMSWFRlFBAcVFBRQXGsUFBRQXhcsKjaJw2+KCjttGYUHKlxkFBUZReL/Ajn4vMI7e7tjWoCBlfafbFtwOWoedn4ultaEKwhakhY8z6PRYM+vc2kxvhVrn/R5pVXY6hh19rB19bMe+e6uxc6s7WN/VcdL/MwiWeaf9ddpXp2P0nG7pa9M3T20Je8p/Kh3/YfR0frpsgffwPNL30fH4zvvM/Lnlg2wE+jY6X6NxAp2vxygDzN3Z3HCQNVv38sqWPazZupc3duzv1BI2g8oRpRw/upQTRpdx6sQKjisvpbysiOElRYwoDQJ6RGkRw0sKGV4SfB9WEiwvKy6ksCD//yCkfzpnZn9+X/S7diyyEeiLgZvD60GeRXCFb/WfD6C29iSrt+xleW3DkQDfc7AVgOElhcyZMJrPfHAas08czYkVZYwbVUZ1eSnFhRqlKpLPeg10M/spsACoMrM6ggv2FgO4+70E14G8BNgAHAQ+NVDFDmV7mg7z3NsJnl6/i+feTrDvUCtmMOO4kVw8axynTRrD3IkVvG9cuVrSIkNUJqNcru1lvQOfzVpFAgTdKG+9e4Cn1+/i6Td3sXrLHpIOVSNLuHjWOC485TjOPamK0cOKoy5VRGJCl6CLoTe27+cLj6xh/c7g6l2zx4/i5g+fxAUzxzFn/GgK1AIXkS4o0GMkmXT+fdlG7nryLUYPL+Ybfzqbi2aOY9yosqhLE5EcoECPiZ37mrn10TU8v6GBj8waxzc/PoexI0qiLktEcogCPQZ+89oObnvsNQ63Jfnmxz7A1WdOHBJjZkUkuxToEWpsaeOOxet4dFUdcyaM5l+unsu06pFRlyUiOUqBHpHVW/bwNz9bw9bdB7n5wydxy0UzNE5cRPpFgR6BHfsOcc19y6keWcrDN57D/Kljoy5JRPKAAj0CS17dweG2JA9+er66WEQka/Q/fgSWvLaD2eNHKcxFJKsU6INs6+6DvLp1L5fOOTHqUkQkzyjQB9mStcG8ZR/9wAkRVyIi+UaBPsiWrN3OqRMrmDh2eNSliEieUaAPoo31Tazbvp8/maPWuYhknwJ9EP167XYALlF3i4gMAAX6IFqydgfzJo/hxIphUZciInlIgT5INuw6wPqdB7hU3S0iMkAU6IPkiVd3YAaL1N0iIgNEgT4I3J0la7czf8pYzW0uIgNGgT4I1u88QE2iiUtP1YeJRGTgKNAHwa/X7qDAYNHs46MuRUTymAJ9gHV0t5w7vYqqkaVRlyMieUyBPsDWbd/PpoaDGt0iIgNOgT7Anli7naIC44/fr+4WERlYCvQB5O78eu0OzjupijG64LOIDDAF+gB6tW4fdXsOqbtFRAaFAn0ALXl1OyWFBXxE3S0iMggU6AMkmXR+/doOPvS+KkYPK466HBEZAhToA2T1lj3s2NfMR9XdIiKDRIE+QJas3UFJUQEXzRwXdSkiMkQo0AdAe9jd8uGTqykvU3eLiAwOBfoAWLV5D4kDLboQtIgMqowC3cwWmtlbZrbBzG7rYv0kM3vGzF4xs7Vmdkn2S80dr2/bB8A50ysjrkREhpJeA93MCoF7gEXALOBaM5uVttlXgUfc/TTgGuB72S40l9TWNzKqrIhKfZhIRAZRJi30+cAGd69198PAw8Dlads4MCq8PRrYnr0Sc09toolp1SMxs6hLEZEhJJNAHw9sTblfFy5L9XXgE2ZWBywF/rqrHZnZjWa20sxWJhKJYyg3N9QkGplWPSLqMkRkiMnWm6LXAj9y9wnAJcCDZvaefbv7fe4+z93nVVdXZ+nQ8dLY0sa7+1uYXj0y6lJEZIjJJNC3ARNT7k8Il6X6NPAIgLu/CJQBVdkoMNdsTDQBMK1KLXQRGVyZBPrLwAwzm2pmJQRvei5O22YLcCGAmc0kCPT87VPpQW19IwDTj1MLXUQGV6+B7u5twM3Ak8CbBKNZ1pnZnWZ2WbjZrcBfmtmrwE+BG9zdB6roOKtJNFFgMLlyeNSliMgQU5TJRu6+lODNztRlt6fcfgM4L7ul5abaRCMTxgyntKgw6lJEZIjRJ0WzLBiyqP5zERl8CvQsSiadjfVNTKtS/7mIDD4Fehbt3N/ModZ2tdBFJBIK9Cyq7RiyqEAXkQgo0LPoyJBFfahIRCKgQM+i2kQTI0oKOa68NOpSRGQIUqBnUTCHiyblEpFoKNCzSEMWRSRKCvQsOXS4nW17D6n/XEQio0DPko31GuEiItFSoGdJxwgXfahIRKKiQM+SjjHoUzVtrohERIGeJbWJRsZXDGNYiSblEpFoKNCzpLZeI1xEJFoK9Cxw92DIorpbRCRCCvQsSBxoobGljWkasigiEVKgZ0GNJuUSkRhQoGdBTSIcsqgWuohESIGeBbWJJoYVF3LCqLKoSxGRIUyBngW19Y1MrRpBQYEm5RKR6CjQs0CTcolIHCjQ+6mlrZ26PQfVfy4ikVOg99PmhoMkHaarhS4iEVOg91NtQpNyiUg8KND7qWMM+lS10EUkYgr0fqpNNDFuVCkjS4uiLkVEhjgFej/V1jequ0VEYkGB3g9HJuVSd4uIxIACvR8amg6z71CrriMqIrGgQO+HWk3KJSIxklGgm9lCM3vLzDaY2W3dbHOVmb1hZuvM7CfZLTOeOoYsqoUuInHQ69AMMysE7gEuBuqAl81ssbu/kbLNDODLwHnuvsfMjhuoguOktr6JkqICTqwYFnUpIiIZtdDnAxvcvdbdDwMPA5enbfOXwD3uvgfA3Xdlt8x4qk00MrVyBIWalEtEYiCTQB8PbE25XxcuS/U+4H1m9ryZLTezhV3tyMxuNLOVZrYykUgcW8UxohEuIhIn2XpTtAiYASwArgV+YGYV6Ru5+33uPs/d51VXV2fp0NFobU+yZfdBBbqIxEYmgb4NmJhyf0K4LFUdsNjdW919I/A2QcDnrS27D9KWdH2oSERiI5NAfxmYYWZTzawEuAZYnLbNLwla55hZFUEXTG32yowfDVkUkbjpNdDdvQ24GXgSeBN4xN3XmdmdZnZZuNmTQIOZvQE8A/ytuzcMVNFxUKvriIpIzGQ0o5S7LwWWpi27PeW2A18Iv4aE2kQTVSNLGT2sOOpSREQAfVL0mNXWN6q7RURiRYF+jGoSTbpKkYjEigL9GOw9eJjdTYc1wkVEYkWBfgxqNMJFRGJIgX4MNMJFROJIgX4MauubKC40Jo7RpFwiEh8K9GNQm2hk0tjhFBXq9IlIfCiRjkFtoklzoItI7CjQ+6itPcmmhib1n4tI7CjQ+6huzyFa210jXEQkdhTofVRb33HZOQW6iMSLAr2PjsyyqA8ViUjMKND7qCbRyNgRJYwZURJ1KSIinSjQ+6gm0cS0KnW3iEj8KND7SNcRFZG4UqD3wf7mVuobWzQGXURiSYHeB0cvO6dAF5H4UaD3Qc2ujkm51OUiIvGjQO+D2vpGigqMSWOHR12KiMh7KND7oDbRxKSxwynWpFwiEkNKpj4IRrio/1xE4kmBnqH2pLOxQdcRFZH4UqBnaNueQxxuS+oNURGJLQV6hmrqddk5EYk3BXqGOsag60NFIhJXCvQM1SQaqRhezFhNyiUiMaVAz1BtolGTcolIrCnQM6QhiyISdwr0DBxobmXXAU3KJSLxpkDPwNFJudTlIiLxpUDPgK4jKiK5IKNAN7OFZvaWmW0ws9t62O7jZuZmNi97JUavNtFEYYExaawCXUTiq9dAN7NC4B5gETALuNbMZnWxXTlwC7Ai20VGrWNSrpIi/UMjIvGVSULNBza4e627HwYeBi7vYrv/A3wLaM5ifbFQoyGLIpIDMgn08cDWlPt14bIjzOx0YKK7/7qnHZnZjWa20sxWJhKJPhcbhWTS2Viv64iKSPz1uw/BzAqAfwJu7W1bd7/P3ee5+7zq6ur+HnpQbNt7iJa2pMagi0jsZRLo24CJKfcnhMs6lAOzgWfNbBNwNrA4X94Yra0Phyyqy0VEYi6TQH8ZmGFmU82sBLgGWNyx0t33uXuVu09x9ynAcuAyd185IBUPstpEOGTxOLXQRSTeeg10d28DbgaeBN4EHnH3dWZ2p5ldNtAFRq0m0ciosiIqNSmXiMRcUSYbuftSYGnastu72XZB/8uKj445XMws6lJERHqkgdW9CAJd/eciEn8K9B40trSxc3+zJuUSkZygQO/BxiNXKVILXUTiT4Heg1pdR1REcogCvQc1iSYKDCZXDo+6FBGRXinQe1CbaGTi2OGUFhVGXYqISK8U6D2oSTTpE6IikjMU6N0IJuVqVP+5iOQMBXo3duxvprk1qTHoIpIzFOjdODKHi1roIpIjFOjdqNnVMWRRLXQRyQ0K9G7U1jdRXlpE9cjSqEsREcmIAr0bHXO4aFIuEckVCvRu1CY0wkVEcosCvQsHD7exfV+z5nARkZyiQO9CbTgpl1roIpJLFOhdOHIdUbXQRSSHKNC7UJtoxAymVCrQRSR3KNC7UJNoYsKYYZQVa1IuEckdCvQuvLVzPyep/1xEcowCPc3upsO8/W4j86aMjboUEZE+UaCneWljAwBnTVWgi0huUaCnWV67m7LiAuZMqIi6FBGRPlGgp1mxcTdnTB5DSZFOjYjkFqVWir0HD7N+537OmloZdSkiIn2mQE/x0sbduKv/XERykwI9xYqNuyktKuDUiRVRlyIi0mcK9BQrNjZw2qQKfaBIRHKSAj2071Ar67ar/1xEcpcCPbRyU9B/fvY0BbqI5KaMAt3MFprZW2a2wcxu62L9F8zsDTNba2b/ZWaTs1/qwFqxcTclhQWcNqki6lJERI5Jr4FuZoXAPcAiYBZwrZnNStvsFWCeu88Bfg78Q7YLHWgrahuYO1H95yKSuzJpoc8HNrh7rbsfBh4GLk/dwN2fcfeD4d3lwITsljmwDjS38tq2fZw1TcMVRSR3ZRLo44GtKffrwmXd+TTwm65WmNmNZrbSzFYmEonMqxxgKzfvIan+cxHJcVl9U9TMPgHMA+7qar273+fu89x9XnV1dTYP3S8randTXGicPmlM1KWIiByzogy22QZMTLk/IVzWiZldBPwv4I/cvSU75Q2OFRsbmDOhgmEl6j8XkdyVSQv9ZWCGmU01sxLgGmBx6gZmdhrwfeAyd9+V/TIHTlNLG2vr9unj/iKS83oNdHdvA24GngTeBB5x93VmdqeZXRZudhcwEnjUzNaY2eJudhc7qzbvoT3p6j8XkZyXSZcL7r4UWJq27PaU2xdlua5Bs2JjA4UFxhmT1X8uIrltyH9SdEXtbj4wfjQjSjN6bRMRia0hHeiHDrfzat1ejT8XkbwwpAN99ZY9tLar/1xE8sOQDvQVtQ0UGMxT/7mI5IEhHejLN+5m9vjRlJcVR12KiEi/DdlAb25tZ82WvRp/LiJ5Y8gG+itb9nK4Pan+cxHJG0M20FdsbMAM5k1RC11E8sOQDfTltQ3MOmEUo4ep/1xE8sOQDPSWtnZe2bJX1w8VkbwyJAP91a37aGlLcrY+UCQieWRIBvqK2qD/fL5GuIhIHhmSgb58YwMnjyunYnhJ1KWIiGTNkAv0/c2trNq8R8MVRSTvDLlAf+D5TTS3JrnyjJy6jrWISK+GVKA3trTxw2UbufCU45g9fnTU5YiIZNWQCvQfv7iJfYda+dyFM6IuRUQk64ZMoDe1tPHDP2xkwcnVnDqxIupyRESybsgE+kMrNrO76TB/fYFa5yKSn4ZEoB863M59v6/lgzOqdO1QEclbQyLQH1qxmfrGw+o7F5G8lvdXRm5ubef7v6/l3OmVnKmZFUUi19raSl1dHc3NzVGXEmtlZWVMmDCB4uLMJxDM+0B/+KUtJA608K/XnhZ1KSIC1NXVUV5ezpQpUzCzqMuJJXenoaGBuro6pk6dmvHj8rrLpbm1nX97rob5U8fqk6EiMdHc3ExlZaXCvAdmRmVlZZ//i8nrQH905Vbe3d/CLeo7F4kVhXnvjuUc5W2gt7S1871na5g3eQznTlfrXETyX94G+i9WbWPHvmY+d+EMtQZE5Ii9e/fyve99r8+Pu+SSS9i7d2+P29x+++089dRTx1hZ/+VloLe2J7nnmQ3MnVjBB2dURV2OiMRId4He1tbW4+OWLl1KRUVFj9vceeedXHTRRf0pr1/ycpTL46u3sW3vIf7+itlqnYvE2B1PrOON7fuzus9ZJ47ia3/y/m7X33bbbdTU1DB37lyKi4spKytjzJgxrF+/nrfffpsrrriCrVu30tzczC233MKNN94IwJQpU1i5ciWNjY0sWrSI888/nxdeeIHx48fzq1/9imHDhnHDDTdw6aWXcuWVVzJlyhSuv/56nnjiCVpbW3n00Uc55ZRTSCQSXHfddWzfvp1zzjmH3/3ud6xatYqqqv43PvOuhd7U0sZ3n9nAnAmjWXByddTliEjMfPOb32T69OmsWbOGu+66i9WrV/Od73yHt99+G4D777+fVatWsXLlSu6++24aGhres4933nmHz372s6xbt46Kigp+8YtfdHmsqqoqVq9ezU033cS3v/1tAO644w4uuOAC1q1bx5VXXsmWLVuy9tzypoW+qb6JH7+4mUdXbuVASxt3XHamWuciMddTS3qwzJ8/v9NY77vvvpvHH38cgK1bt/LOO+9QWdl5YMXUqVOZO3cuAGeccQabNm3qct8f+9jHjmzz2GOPAbBs2bIj+1+4cCFjxmRvOpKMAt3MFgLfAQqBH7r7N9PWlwI/Bs4AGoCr3X1T1qrshrvzh3fq+dELm3jmrV0UmvHROSdww7lTOG2S5mwRkd6NGDHiyO1nn32Wp556ihdffJHhw4ezYMGCLseCl5aWHrldWFjIoUOHutx3x3aFhYW99tFnQ6+BbmaFwD3AxUAd8LKZLXb3N1I2+zSwx91PMrNrgG8BVw9EwRBcqOKx1XU88MImahJNVI0s5a8vmMGfnzWJcaPKBuqwIpIHysvLOXDgQJfr9u3bx5gxYxg+fDjr169n+fLlWT/+eeedxyOPPMKXvvQlfvvb37Jnz56s7TuTFvp8YIO71wKY2cPA5UBqoF8OfD28/XPgu2Zm7u5ZqzT0s5e38PdL3uRASxunThjNP199Kpd84ARKiwqzfSgRyUOVlZWcd955zJ49m2HDhjFu3Lgj6xYuXMi9997LzJkzOfnkkzn77LOzfvyvfe1rXHvttTz44IOcc845HH/88ZSXl2dl39Zb5prZlcBCd/9MeP+TwFnufnPKNq+H29SF92vCberT9nUjcCPApEmTzti8eXOfC172Tj2PrtqqbhWRHPXmm28yc+bMqMuITEtLC4WFhRQVFfHiiy9y0003sWbNmi637epcmdkqd5/X1faD+qaou98H3Acwb968Y2q9nz+jivM1tlxEctSWLVu46qqrSCaTlJSU8IMf/CBr+84k0LcBE1PuTwiXdbVNnZkVAaMJ3hwVEZEUM2bM4JVXXhmQfWcyDv1lYIaZTTWzEuAaYHHaNouB68PbVwJPD0T/uYjkB8VD747lHPUa6O7eBtwMPAm8CTzi7uvM7E4zuyzc7N+BSjPbAHwBuK3PlYjIkFBWVkZDQ4NCvQcd86GXlfVt1F6vb4oOlHnz5vnKlSsjObaIREdXLMpMd1csis2boiIixcXFfboKj2Qu7+ZyEREZqhToIiJ5QoEuIpInIntT1MwSQN8/KhqoAup73SpauVAj5EadqjE7VGN2RF3jZHfvcm7wyAK9P8xsZXfv8sZFLtQIuVGnaswO1Zgdca5RXS4iInlCgS4ikidyNdDvi7qADORCjZAbdarG7FCN2RHbGnOyD11ERN4rV1voIiKSRoEuIpInci7QzWyhmb1lZhvMLJazOprZJjN7zczWmFksZiAzs/vNbFd4damOZWPN7Hdm9k74PdJLQHVT49fNbFt4LteY2SUR1zjRzJ4xszfMbJ2Z3RIuj8257KHGuJ3LMjN7ycxeDeu8I1w+1cxWhH/jPwun7Y5bjT8ys40p53JuVDV24u458wUUAjXANKAEeBWYFXVdXdS5CaiKuo60mj4EnA68nrLsH4Dbwtu3Ad+KYY1fB74Y9flLqecE4PTwdjnwNjArTueyhxrjdi4NGBneLgZWAGcDjwDXhMvvBW6KYY0/Aq6M+hymf+VaC/3IBavd/TDQccFq6YW7/x7Ynbb4cuCB8PYDwBWDWVO6bmqMFXff4e6rw9sHCK4RMJ4YncseaowVDzSGd4vDLwcuILjYPER/LrurMZZyLdDHA1tT7tcRw19Ugh/4b81sVXhh7Lga5+47wts7gXE9bRyhm81sbdglE5srg5vZFOA0glZbLM9lWo0Qs3NpZoVmtgbYBfyO4D/wvR5cWAdi8DeeXqO7d5zLb4Tn8p/NrDS6Co/KtUDPFee7++nAIuCzZvahqAvqjQf/U8ax5fFvwHRgLrAD+MdIqwmZ2UjgF8Dn3X1/6rq4nMsuaozduXT3dnefS3Ct4vnAKdFW9F7pNZrZbODLBLWeCYwFvhRdhUflWqBncsHqyLn7tvD7LuBxgl/UOHrXzE4ACL/virie93D3d8M/qCTwA2JwLs2smCAoH3L3x8LFsTqXXdUYx3PZwd33As8A5wAV4cXmIUZ/4yk1Lgy7tdzdW4D/ICbnMtcCPZMLVkfKzEaYWXnHbeAjwOs9PyoyqRf3vh74VYS1dKkjJEN/SsTn0syM4Bq6b7r7P6Wsis257K7GGJ7LajOrCG8PAy4m6O9/huBi8xD9ueyqxvUpL95G0Mcfi7/xnPukaDjU6l8IRrzc7+7fiLaizsxsGkGrHIJL/P0kDjWa2U+BBQRTf74LfA34JcGIgkkEUxlf5e6RvSnZTY0LCLoInGD00F+l9FUPOjM7H/gD8BqQDBd/haCPOhbnsocaryVe53IOwZuehQSNy0fc/c7wb+hhgq6MV4BPhC3hONX4NFBNMApmDfA/Ut48jUzOBbqIiHQt17pcRESkGwp0EZE8oUAXEckTCnQRkTyhQBcRyRMKdJFjYGYLzGxJ1HWIpFKgi4jkCQW65DUz+0Q4n/UaM/t+ONFSYzih0joz+y8zqw63nWtmy8MJlx7vmLzKzE4ys6fCObFXm9n0cPcjzeznZrbezB4KPzUoEhkFuuQtM5sJXA2cF06u1A78OTACWOnu7weeI/hEKsCPgS+5+xyCT1l2LH8IuMfdTwXOJZjYCoJZDD9PMNf4NOC8AX5KIj0q6n0TkZx1IXAG8HLYeB5GMGlWEvhZuM1/Ao+Z2Wigwt2fC5c/ADwazssz3t0fB3D3ZoBwfy+5e114fw0wBVg24M9KpBsKdMlnBjzg7l/utNDsf6dtd6zzX6TOL9KO/p4kYupykXz2X8CVZnYcHLnu52SC3/uO2fyuA5a5+z5gj5l9MFz+SeC58Io/dWZ2RbiPUjMbPphPQiRTalFI3nL3N8zsqwRXjyoAWoHPAk0EFyr4KkEXzNXhQ64H7g0Duxb4VLj8k8D3zezOcB9/NohPQyRjmm1Rhhwza3T3kVHXIZJt6nIREckTaqGLiOQJtdBFRPKEAl1EJE8o0EVE8oQCXUQkTyjQRUTyxH8D8WGWbYVsTTQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEWCAYAAABPON1ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdlElEQVR4nO3de3Bc5Znn8e/T3bpY6rZl62aM71gyxlxkY4iJDcUSkiWXJSxJGEigkuwkTFKTGTKTrQ3MJCFQM7WZ2WQ2SW2AmMskmxAIS66bolhCghOcEMCACAYbZIOxZWNbvsiWbEuW1M/+0UeyLGS5ZXfrnO7+fapU3X369NHTx9ZPr95+z/uauyMiItEVC7sAEREZm4JaRCTiFNQiIhGnoBYRiTgFtYhIxCmoRUQiTkEtIhJxCmopaGa22cwuD7sOkXxSUIuIRJyCWoqOmVWY2TfNbHvw9U0zqwieqzOzX5lZp5ntNbMnzSwWPPdFM9tmZl1m9qqZvSvcdyKSkQi7AJE8+EdgOdACOPAL4EvAl4EvAO1AfbDvcsDNbCHwOeACd99uZnOB+MSWLTI6tailGH0MuN3dd7l7B3AbcEPwXB9wGjDH3fvc/UnPTHgzAFQAZ5lZmbtvdvdNoVQvMoKCWorRDODNYY/fDLYB/A9gI/CYmb1uZjcDuPtG4PPAV4FdZvagmc1AJAIU1FKMtgNzhj2eHWzD3bvc/QvuPh+4Evj7wb5od/+Ru68MXuvAv0xs2SKjU1BLMSgzs8rBL+AB4EtmVm9mdcBXgB8CmNkHzGyBmRmwn0yXR9rMFprZZcGHjj3AYSAdztsROZaCWorBI2SCdfCrElgL/Bl4CXge+Kdg3ybgcaAbeAq4w92fINM//TVgN7ADaABumbi3IHJ8poUDRESiTS1qEZGIU1CLiEScglpEJOIU1CIiEZeXS8jr6up87ty5+Ti0iEhReu6553a7e/1oz+UlqOfOncvatWvzcWgRkaJkZm8e7zl1fYiIRJyCWkQk4hTUIiIRp/moRSQn+vr6aG9vp6enJ+xSIq2yspKZM2dSVlaW9WsU1CKSE+3t7aRSKebOnUtmzisZyd3Zs2cP7e3tzJs3L+vXqetDRHKip6eH2tpahfQYzIza2tpx/9WhoBaRnFFIn9jJnKNIBfW3f9PGum37wy5DRCRSIhPUnYeO8MAzW7j6zj/yo6e3oOlXRWS8Ojs7ueOOO8b9uve97310dnaOuc9XvvIVHn/88ZOs7NREJqhrqsr51d+s5B3zpvEPP3uJv3/oRQ4d6Q+7LBEpIMcL6v7+sbPkkUceoaamZsx9br/9di6//PJTKe+kRSaoAWqTFXzvkxfyd5c38/PWbVz1nT+wcVdX2GWJSIG4+eab2bRpEy0tLVxwwQVcfPHFXHnllZx11lkAXHXVVZx//vksXryYVatWDb1u7ty57N69m82bN7No0SI+/elPs3jxYt7znvdw+PBhAD7xiU/w8MMPD+1/6623snTpUs455xw2bNgAQEdHB+9+97tZvHgxn/rUp5gzZw67d+8+5fcVueF58Zhx0+VNLJ1Tw00PtnLl//oD//3qc/hgy+lhlyYiWbrt/77MK9sP5PSYZ82YzK3/afGY+3zta19j3bp1tLa2snr1at7//vezbt26oaFw9913H9OmTePw4cNccMEFfOhDH6K2tvaYY7S1tfHAAw9w9913c8011/CTn/yE66+//m3fq66ujueff5477riDr3/969xzzz3cdtttXHbZZdxyyy08+uij3HvvvTl575FqUQ93cVM9j/ztxZx12mRuerCVL/98Hb39A2GXJSIF5MILLzxmvPK3v/1tzjvvPJYvX87WrVtpa2t722vmzZtHS0sLAOeffz6bN28e9dhXX3312/ZZs2YN1157LQBXXHEFU6dOzcn7iFyLerjpUyp54Mbl/OujG7j7yTd4sb2T73x0KbOmVYVdmoiM4UQt34lSXV09dH/16tU8/vjjPPXUU1RVVXHppZeOOp65oqJi6H48Hh/q+jjefvF4/IR94Kcqsi3qQWXxGP/4/rP47g3n88bug3z+x61hlyQiEZVKpejqGv1zrf379zN16lSqqqrYsGEDf/rTn3L+/VesWMFDDz0EwGOPPca+fftyctxIt6iH+4+Lp/PK9gN8+7dt7D/Ux5Sq7K+TF5HSUFtby4oVKzj77LOZNGkSjY2NQ89dccUV3HXXXSxatIiFCxeyfPnynH//W2+9leuuu44f/OAHXHTRRUyfPp1UKnXKx7V8jFdetmyZ52PhgGc37+Ujdz3FnR9bynvPOS3nxxeRk7d+/XoWLVoUdhmh6u3tJR6Pk0gkeOqpp/jsZz9La2vr2/Yb7VyZ2XPuvmy04xZMixqgZVYNyYoEazbuVlCLSORs2bKFa665hnQ6TXl5OXfffXdOjltQQV0Wj7F8/jTWbDz1cYkiIrnW1NTECy+8kPPjRv7DxJFWLKjjzT2H2Lr3UNiliMgImvrhxE7mHBVcUF/cVAegVrVIxFRWVrJnzx6F9RgG56OurKwc1+sKqusD4Iz6JI2TK1izcTfXXTg77HJEJDBz5kza29vp6OgIu5RIG1zhZTwKLqjNjJUL6vnthp2k004spvlvRaKgrKxsXKuWSPYKrusDYGVTLfsO9fFyjucSEBGJooIM6hUL1E8tIqWjIIO6IVXJmdNTrNmovjARKX4FGdSQaVU/u3kfPX2aUU9EilvBBvXKpjqO9Kd5dvPesEsREcmrgg3qd8ybRlnc1E8tIkWvYIO6qjzB0tlTWdOmoBaR4lawQQ2wckEdL28/wN6DR8IuRUQkbwo7qIPLyf+g7g8RKWIFHdTnzqwhVZlQ94eIFLWCDup4zHjnGbWs2bhbE8GISNEq6KAGWNlUz7bOw2zeo2lPRaQ4FX5Q63JyESlyWQW1mf2dmb1sZuvM7AEzG99kqnk0t7aK02smsaZNl5OLSHE6YVCb2enA3wLL3P1sIA5cm+/CspWZ9rSOP27aw0Ba/dQiUnyy7fpIAJPMLAFUAdvzV9L4rWyqo6unnz+3d4ZdiohIzp0wqN19G/B1YAvwFrDf3R8buZ+Z3Whma81s7USv8DA47anGU4tIMcqm62Mq8EFgHjADqDaz60fu5+6r3H2Zuy+rr6/PfaVjmFZdzuIZk3lS46lFpAhl0/VxOfCGu3e4ex/wU+Cd+S1r/FY21fH8ln0c7O0PuxQRkZzKJqi3AMvNrMrMDHgXsD6/ZY3fygV19A04z2jaUxEpMtn0UT8NPAw8D7wUvGZVnusatwvmTqM8EdPl5CJSdLJahdzdbwVuzXMtp6SyLM4Fc6fqA0URKToFf2XicBfNr2XDji66evrCLkVEJGeKKqjPnD4ZgLZd3SFXIiKSO0UV1AunpwB4bUdXyJWIiOROUQX16TWTqCqP8+pOBbWIFI+iCupYzGhqTPGaglpEikhRBTXAwsYkr+5QH7WIFI+iC+rmxhS7u3vZ090bdikiIjlRdEE99IHiTrWqRaQ4FF9QNw4GtfqpRaQ4FF1Q16cqqKkq08gPESkaRRfUZkZzY0pjqUWkaBRdUEOm++PVnV24a2kuESl8RRnUzdNTdPX0s+NAT9iliIicsqIM6sEPFF9V94eIFIGiDOrmxiSgkR8iUhyKMqhrqsppnFyhKxRFpCgUZVBD5gpFtahFpBgUbVAvbEzRtquLgbRGfohIYSvaoG6enqKnL83WvYfCLkVE5JQUbVAPjfxQ94eIFLiiDeqmwZEfGqInIgWuaIO6qjzB7GlValGLSMEr2qAGjfwQkeJQ1EG9cHqS1zsOcqQ/HXYpIiInraiDurkxRX/aeWP3wbBLERE5aUUd1IOrvaifWkQKWVEH9fy6JImYaeSHiBS0og7q8kSMeXXValGLSEEr6qCGzBWKGvkhIoWs6IN6YWOKLXsPcehIf9iliIiclKIP6ubGFO6wcZemPBWRwlT0QT008kMfKIpIgSr6oJ49rYqKREz91CJSsLIKajOrMbOHzWyDma03s4vyXViuxGPGgoYkr+5U14eIFKZsW9TfAh519zOB84D1+Ssp9xY2pjSWWkQK1gmD2symAJcA9wK4+xF378xzXTnVPD3FjgM97D/UF3YpIiLjlk2Leh7QAfy7mb1gZveYWfXInczsRjNba2ZrOzo6cl7oqRhcROC1XWpVi0jhySaoE8BS4E53XwIcBG4euZO7r3L3Ze6+rL6+PsdlnppmjfwQkQKWTVC3A+3u/nTw+GEywV0wZkypJFmR0MgPESlIJwxqd98BbDWzhcGmdwGv5LWqHDMzmhuTalGLSEFKZLnf3wD3m1k58DrwyfyVlB8Lp6d4dN0O3B0zC7scEZGsZTU8z91bg/7nc939Knffl+/Ccq25McW+Q310dPeGXYqIyLgU/ZWJg4ZGfuzQhS8iUlhKJqibtdqLiBSokgnqumQFtdXlukJRRApOyQQ1ZPqp1aIWkUJTUkG9cHqKtp1dpNMedikiIlkrqaBubkxx8MgA2zoPh12KiEjWSiqoF05PAugKRREpKCUV1AsaMiM/2rQsl4gUkJIK6imTymicXKEWtYgUlJIKaoCmhpQWuhWRglJ6Qd2YpG1nt0Z+iEjBKL2gbkhxuE8jP0SkcJRcUDc3ZkZ+tGm1FxEpECUX1E2DIz+0KrmIFIiSC+opVWU0pCp4TUEtIgWi5IIagg8U1fUhIgWiNIM6GKKnkR8iUghKMqibG1Mc0pwfIlIgSjKom4KRH7rwRUQKQWkGdYMmZxKRwlGSQV1TVU59qkKTM4lIQSjJoIbMhS9talGLSAEo2aBuakjRtqsbd438EJFoK92gbkxq5IeIFITSDWotIiAiBaKEgzqYnEn91CIScSUb1FOry6lLVmhyJhGJvJINasiM/HhNXR8iEnElHdRNDUk27uzSyA8RibTSDurGFAePDLB9f0/YpYiIHFdpB7U+UBSRAlDSQd3cqNVeRCT6sg5qM4ub2Qtm9qt8FjSRhkZ+aBEBEYmw8bSobwLW56uQsDQ1JLUsl4hEWlZBbWYzgfcD9+S3nInX3Jhko+b8EJEIy7ZF/U3gvwHp4+1gZjea2VozW9vR0ZGL2ibEgsYU3b39vKWRHyISUScMajP7ALDL3Z8baz93X+Xuy9x9WX19fc4KzLfmwZEfuvBFRCIqmxb1CuBKM9sMPAhcZmY/zGtVE6hpaOSHPlAUkWg6YVC7+y3uPtPd5wLXAr919+vzXtkEmVZdTl2yXEP0RCSySnoc9aAFDUle0xA9EYmocQW1u6929w/kq5iwNDem2LhTIz9EJJrUoiYzlrqrt58dBzTyQ0SiR0HN0Q8UdeGLiESRghpNziQi0aagBmqTFdRWa+SHiESTgjqwoCGpyZlEJJIU1IHmxhRtGvkhIhGkoA40N2ZGfuw80Bt2KSIix1BQBxY0DI78UPeHiESLgjrQ3KjJmUQkmhTUgdpkBdOqyzVET0QiR0E9TFNDUi1qEYkcBfUwTY1JXtvZpZEfIhIpCuphmhtTdPX0s6tLIz9EJDoU1MMsCC4l18gPEYkSBfUwzcHkTK/uUFCLSHQoqIepS1YwY0olL7bvD7sUEZEhCuoRWmbX0Lp1X9hliIgMUVCPsGTWVLbuPczubn2gKCLRoKAeoWV2DQCtWzpDrUNEZJCCeoSzZ0whETNeUPeHiESEgnqESeVxzjwtRevWzrBLEREBFNSjWjJrKi9u3c9AWlcoikj4FNSjaJlVQ3dvP5s6NO+HiIRPQT2KwQ8UX9iifmoRCZ+CehTzaquZMqlM/dQiEgkK6lHEYsZ5s2p4QUP0RCQCFNTHsWRWDa/t7KK7tz/sUkSkxCmoj6Nldg1phz+3d4ZdioiUOAX1cbTMrAFQP7WIhE5BfRxTq8uZV1etfmoRCZ2CegxLZtXQurVTS3OJSKgU1GNomV1DR1cv2zoPh12KiJQwBfUYlsyaCqifWkTCdcKgNrNZZvaEmb1iZi+b2U0TUVgUnHlaiopETFOeikioElns0w98wd2fN7MU8JyZ/drdX8lzbaEri8c45/QpvKAWtYiE6IQtand/y92fD+53AeuB0/NdWFS0zKph3bb9HOlPh12KiJSocfVRm9lcYAnw9CjP3Whma81sbUdHR47KC9+S2VPp7U+zYceBsEsRkRKVdVCbWRL4CfB5d39barn7Kndf5u7L6uvrc1ljqIaW5lL3h4iEJKugNrMyMiF9v7v/NL8lRcuMKZXUpyp04YuIhCabUR8G3Ausd/d/y39J0WJmQxe+iIiEIZsW9QrgBuAyM2sNvt6X57oipWV2DW/sPsi+g0fCLkVEStAJh+e5+xrAJqCWyBq68KW9k/+wsCHkakSk1OjKxCycO3MKMUP91CISCgV1FqorEjQ3ptRPLSKhUFBnacnsGlq37COd1kx6IjKxFNRZWjJrKgd6+nljz8GwSxGREqOgztLghS/qpxaRiaagztKC+iSpigStW/eFXYqIlBgFdZZiMePcWVPUohaRCaegHocls6ayYUcXh48MhF2KiJQQBfU4tMyqYSDtvLRtf9iliEgJUVCPw9GZ9NRPLSITR0E9DnXJCmZPq+LJtt1hlyIiJURBPU4fPn8mT7bt5tUdXWGXIiIlQkE9Tjcsn8Oksjjf/f2msEsRkRKhoB6nqdXlXHvhLH7Zup1tnYfDLkdESoCC+iT85cp5OHDfmjfCLkVESoCC+iTMnFrFlefN4IFnttB5SIsJiEh+KahP0o2XzOfQkQF++Kc3wy5FRIqcgvokLTptMpcurOd7f9xMT5+uVBSR/FFQn4K/uuQMdncf4eHn2sMuRUSKmIL6FCyfP43zZtVw95OvM6AFBUQkTxTUp8DM+Mwl83lzzyEeXbcj7HJEpEgpqE/RexZPZ15dNXf9bhPualWLSO4pqE9RPGZ8+uL5vLRtP09t2hN2OSJShBTUOXD10tOpS1Zw1+9fD7sUESlCCuocqCyL88kVc/n9ax28sv1A2OWISJFRUOfI9cvnUF2uyZpEJPcU1DkyZVIZH33HbH7157fYuvdQ2OWISBFRUOfQf1k5j5jBHas1AkREckdBnUOnTZnER5bN4oFntvDebz3JL1q30T+QDrssESlwCuocu+3KxXzjI+fRn3ZuerCVy77xO+5/+k3NByIiJ83y8Sf6smXLfO3atTk/biFJp51fr9/JHas38eLWTupTFfzlynl87B2zSVWWhV2eiESMmT3n7stGfU5BnV/uzlOb9nDH6k2s2bibyZUJrrtwNktm17CgIcWc2irK4vrDRqTUjRXUiYkuptSYGe9cUMc7F9Tx4tZO7ly9iVVPvs7g78eyuDGvrpqmhhQLGpI0NSZZ0JBkRs0kUhUJzCzcNyAiocuqRW1mVwDfAuLAPe7+tbH2V4t6bAd7+9nU0U3bzm7adnWzcVcXbbu62bL3EMP/OSaVxWmYXEFjqpL64LZxcgUNkyuoqSpncmWCZEUZycoEqcoE1eUJ4jEFu0ghOqUWtZnFge8A7wbagWfN7Jfu/kpuyywd1RUJzp1Zw7kza47Z3tM3wKaObjZ1HGTn/h52HuhhZ1cvuw708Mr2AzxxYBeHjoz9oWSyIkGyIkFVRZyKRJyKRCzzVRanMritSMQoT8Qojx+9LYvHKEvY0LayeIxEzEjEjURs8P7RbfFYZnvmNvN4+P1ELEYsBjHLPI6ZEbPM3CixYY8NY/CPBgseD96PWeaRGfrLQkpaNl0fFwIb3f11ADN7EPggoKDOscqyOItnTGHxjCnH3ae7t5+dB3rYf7iPrp5+unv66erpo7u3n66ezFd3bx8HjwxwpD9Nb3+a3r4BDhzuo6M/TW//AL19mdsj/Wn6BpwjA+nIz6c9GNyxILSNIMiH5bcds78d3WYEgW/BL4Ojx7Bgh6FfFsO+3/BfGsfU8rbabMTjEbfDXnG8ek/ViX6RnfB7ReD34ESUkO9f+NOqynnoMxfl/LjZBPXpwNZhj9uBd4zcycxuBG4EmD17dk6Kk7dLViRI1idzftyBtNM3kObIQJq+/sxt/4DTn3YG0plAH9wnc5t5POCZ5/uPeZx5Pu1OOtiW9sxImIF0sD3Y5g6OH9Pl4555nB72nA8eY+h1mWMcfc2w1w/bNvzY7o6P2O7HvNaPvm7wNRz7C2xkT6GP2D60/7E3Q99/tNedanacqPfyRL+Cs+n+dPIbpBPSTJiAb5KqzM/Hfjk7qruvAlZBpo86V8eViZHpuohTWRYPuxQRGSGbcWHbgFnDHs8MtomIyATIJqifBZrMbJ6ZlQPXAr/Mb1kiIjLohF0f7t5vZp8D/h+Z4Xn3ufvLea9MRESALPuo3f0R4JE81yIiIqPQtcsiIhGnoBYRiTgFtYhIxCmoRUQiLi/TnJpZB/DmSb68Dtidw3LyQTXmhmrMjUKoEQqjzjBrnOPu9aM9kZegPhVmtvZ4M0hFhWrMDdWYG4VQIxRGnVGtUV0fIiIRp6AWEYm4KAb1qrALyIJqzA3VmBuFUCMURp2RrDFyfdQiInKsKLaoRURkGAW1iEjERSaozewKM3vVzDaa2c1h13M8ZrbZzF4ys1Yzi8QKvmZ2n5ntMrN1w7ZNM7Nfm1lbcDs1gjV+1cy2Beey1czeF3KNs8zsCTN7xcxeNrObgu2ROZdj1BiZc2lmlWb2jJm9GNR4W7B9npk9HfyM/ziYNjlqNX7PzN4Ydh5bwqrxGJllj8L9IjN96iZgPlAOvAicFXZdx6l1M1AXdh0jaroEWAqsG7btX4Gbg/s3A/8SwRq/CvzXsM/fsHpOA5YG91PAa8BZUTqXY9QYmXNJZtWuZHC/DHgaWA48BFwbbL8L+GwEa/we8OGwz+HIr6i0qIcW0HX3I8DgArqSBXf/PbB3xOYPAt8P7n8fuGoiaxrpODVGiru/5e7PB/e7gPVk1gyNzLkco8bI8Izu4GFZ8OXAZcDDwfawz+PxaoykqAT1aAvoRuo/3zAOPGZmzwUL+kZVo7u/FdzfATSGWcwYPmdmfw66RkLtnhnOzOYCS8i0tCJ5LkfUCBE6l2YWN7NWYBfwazJ/MXe6e3+wS+g/4yNrdPfB8/jPwXn8n2ZWEV6FR0UlqAvJSndfCrwX+GszuyTsgk7EM3/fRbG1cCdwBtACvAV8I9RqAmaWBH4CfN7dDwx/LirncpQaI3Uu3X3A3VvIrLF6IXBmmPWMZmSNZnY2cAuZWi8ApgFfDK/Co6IS1AWzgK67bwtudwE/I/OfMIp2mtlpAMHtrpDreRt33xn8sKSBu4nAuTSzMjIBeL+7/zTYHKlzOVqNUTyXAO7eCTwBXATUmNngqlKR+RkfVuMVQdeSu3sv8O9E5DxGJagLYgFdM6s2s9TgfeA9wLqxXxWaXwIfD+5/HPhFiLWMajD8Av+ZkM+lmRlwL7De3f9t2FOROZfHqzFK59LM6s2sJrg/CXg3mb70J4APB7uFfR5Hq3HDsF/IRqYPPRI/35G5MjEYTvRNji6g+8/hVvR2ZjafTCsaMutN/igKdZrZA8ClZKZo3AncCvyczKfss8lMOXuNu4f2Yd5xaryUzJ/qTmY0zV8N6wuecGa2EngSeAlIB5v/gUwfcCTO5Rg1XkdEzqWZnUvmw8I4mcbgQ+5+e/Dz8yCZLoUXgOuDlmuUavwtUE9mVEgr8JlhHzqGJjJBLSIio4tK14eIiByHglpEJOIU1CIiEaegFhGJOAW1iEjEKahFhjGzS83sV2HXITKcglpEJOIU1FKQzOz6YD7hVjP7bjDBTncwkc7LZvYbM6sP9m0xsz8FE+38bHDCIjNbYGaPB3MSP29mZwSHT5rZw2a2wczuD65SEwmNgloKjpktAv4CWBFMqjMAfAyoBta6+2Lgd2SufgT438AX3f1cMlf0DW6/H/iOu58HvJPMZEaQmZHu82TmeZ4PrMjzWxIZU+LEu4hEzruA84Fng8buJDITJaWBHwf7/BD4qZlNAWrc/XfB9u8D/yeYs+V0d/8ZgLv3AATHe8bd24PHrcBcYE3e35XIcSiopRAZ8H13v+WYjWZfHrHfyc6PMHz+iQH0cyIhU9eHFKLfAB82swYYWtNwDpn/z4Ozs30UWOPu+4F9ZnZxsP0G4HfB6ijtZnZVcIwKM6uayDchki21FKTguPsrZvYlMivtxIA+4K+Bg2QmgP8Sma6Qvwhe8nHgriCIXwc+GWy/Afiumd0eHOMjE/g2RLKm2fOkaJhZt7snw65DJNfU9SEiEnFqUYuIRJxa1CIiEaegFhGJOAW1iEjEKahFRCJOQS0iEnH/HwD0TCSnUJvLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fold 0: 32\n",
    "# fold 1: 29\n",
    "# fold 2: 31\n",
    "# fold 3: 31\n",
    "# avg: (32+29+31+31)/4 = 30.75\n",
    "# total epoch: 30.75 + 30.75*25/100 = 38.4375 = 38\n",
    "\n",
    "with strategy.scope():\n",
    "    model = effb3()\n",
    "    model.compile(\n",
    "        optimizer = tfa.optimizers.LAMB(LR),\n",
    "        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n",
    "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n",
    "    )\n",
    "model.summary()\n",
    "    \n",
    "model.fit(\n",
    "    train_data, \n",
    "    steps_per_epoch=train_size//BATCH_SIZE,\n",
    "    epochs = 38,\n",
    "    callbacks=[get_lr_callback()], verbose=1\n",
    ")\n",
    "\n",
    "h = model.history\n",
    "plt_acc(h)\n",
    "plt_loss(h)\n",
    "\n",
    "model.save_weights('model_weights.h5')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 17977.435317,
   "end_time": "2021-05-10T01:58:17.749704",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-05-09T20:58:40.314387",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}