cnn_rnn_countrycodes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "\n",
    "from fastai import *\n",
    "from fastai.vision import *\n",
    "import pretrainedmodels\n",
    "from fastai.callbacks.tracker import *\n",
    "from fastai.callbacks.hooks import *\n",
    "from torch.utils.data import Sampler, BatchSampler\n",
    "import json\n",
    "\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The below is useful if you would like to get updates in your notebook on training as you would in console.\n",
    "\n",
    "My workflow is -> fire up an instance -> start jupyter notebook in tmux -> start training -> disconnect\n",
    "\n",
    "(this gives you one update per epoch - with the regular progress bar chances are you will hit a time out before all the updates that were buffered get forwarded to your browser upon reconnection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import fastai\n",
    "# from fastprogress import force_console_behavior\n",
    "# import fastprogress\n",
    "# fastprogress.fastprogress.NO_BAR = True\n",
    "# master_bar, progress_bar = force_console_behavior()\n",
    "# fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('data/txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Create ImageDataBunch using the data block API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sz_lrg, sz_small = 192, 96\n",
    "bs = 520"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "218"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_stats_lrg = pd.read_pickle(f'data/batch_stats_{sz_lrg}.pkl')\n",
    "batch_stats_lrg = batch_stats_lrg[0][:, None, None], batch_stats_lrg[1][:, None, None]\n",
    "\n",
    "batch_stats_small = pd.read_pickle(f'data/batch_stats_{sz_small}.pkl')\n",
    "batch_stats_small = batch_stats_small[0][:, None, None], batch_stats_small[1][:, None, None]\n",
    "\n",
    "country2code = {v: k for k, v in enumerate(pd.read_pickle('data/countrycodes.pkl'))}\n",
    "codes_count = len(country2code); codes_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_max_len = 68\n",
    "\n",
    "def create_func(path):\n",
    "    with open(path) as f: j = json.load(f)\n",
    "    \n",
    "    drawing_lrg = list2drawing(j['drawing'], size=sz_lrg, time_color=True)\n",
    "    tensor_lrg = drawing2tensor(drawing_lrg)\n",
    "    tensor_lrg.div_(255).sub_(batch_stats_lrg[0]).div_(batch_stats_lrg[1])\n",
    "    \n",
    "    drawing_small = list2drawing(j['drawing'], size=sz_small, time_color=True)\n",
    "    tensor_small = drawing2tensor(drawing_small)\n",
    "    tensor_small.div_(255).sub_(batch_stats_small[0]).div_(batch_stats_small[1])\n",
    "    \n",
    "    ary = drawing2seq(j['drawing'])\n",
    "    seq = np.zeros((seq_max_len, 3))\n",
    "    seq[-ary.shape[0]:, :] = ary[:seq_max_len, :]\n",
    "    \n",
    "    country_code = country2code[j['countrycode']] if j['countrycode'] in country2code.keys() else 0\n",
    "    return tensor_lrg, tensor_small, seq.astype(np.float32), country_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10min 2s, sys: 4min 58s, total: 15min 1s\n",
      "Wall time: 20min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "item_list = ItemList.from_folder(PATH/'train', create_func=create_func)\n",
    "item_lists = item_list.random_split_by_pct(0.002, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label_lists = item_lists.label_from_folder()\n",
    "# pd.to_pickle(label_lists.train.y.classes, 'data/classes.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = pd.read_pickle('data/classes.pkl')\n",
    "\n",
    "label_lists = item_lists.label_from_folder(classes=classes)\n",
    "\n",
    "test_items = ItemList.from_folder(PATH/'test', create_func=create_func)\n",
    "label_lists.add_test(test_items);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(\n",
    "    label_lists.train,\n",
    "    num_workers=64,\n",
    "    batch_sampler=BatchSampler(RandomSamplerWithEpochSize(label_lists.train, 200_000), bs, True)\n",
    ")\n",
    "valid_dl = DataLoader(label_lists.valid, bs, False, num_workers=12)\n",
    "test_dl = DataLoader(label_lists.test, bs, False, num_workers=12)\n",
    "\n",
    "data_bunch = ImageDataBunch(train_dl, valid_dl, test_dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = 'resnet50-incv4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBlock(nn.Module):\n",
    "    def __init__(self, c_in, c_out, k):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv1d(c_in, c_out, k)\n",
    "        self.bn = nn.BatchNorm1d(c_out)\n",
    "    def forward(self, x):\n",
    "        x = self.conv(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.bn(x)\n",
    "        return x\n",
    "\n",
    "class RNNBlock(nn.Module):\n",
    "    def __init__(self, inp_sz, hidden_sz=128, num_l=1, bi=False, last_h=False):\n",
    "        super().__init__()\n",
    "        self.rnn = nn.LSTM(input_size=inp_sz, hidden_size=hidden_sz, num_layers=num_l, bidirectional=bi)\n",
    "        self.last_h = last_h\n",
    "    def forward(self, x):\n",
    "        x = self.rnn(x)[0]\n",
    "        if self.last_h: x = x[-1]\n",
    "        x = F.relu(x)\n",
    "        return x\n",
    "\n",
    "cnn_lstm = nn.Sequential(*[\n",
    "    Lambda(lambda x: x.transpose(2,1)),\n",
    "    ConvBlock(3, 48, 5),\n",
    "    ConvBlock(48, 64, 5),\n",
    "    ConvBlock(64, 96, 5),\n",
    "    Lambda(lambda x: x.permute(2,0,1)),\n",
    "    RNNBlock(96),\n",
    "    RNNBlock(128, last_h=True),\n",
    "    Lambda(lambda x: x.squeeze()),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "COUNTRY_EMB_SZ = 20\n",
    "\n",
    "class MixedInputModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv_lrg = nn.Sequential(create_body(models.resnet50(True), -2), nn.AdaptiveAvgPool2d(1))\n",
    "        self.conv_small = create_body(pretrainedmodels.inceptionv4(), -2)\n",
    "        self.cnn_lstm = cnn_lstm\n",
    "        self.country_emb = nn.Embedding(codes_count, COUNTRY_EMB_SZ)\n",
    "        head_inp_sz = num_features_model(self.conv_lrg) + \\\n",
    "            model_sizes(self.conv_small, (96, 96), False)[-1][1] + self.country_emb.embedding_dim + cnn_lstm[6].rnn.hidden_size\n",
    "#         self.head = create_head(head_inp_sz, 340)[2:]\n",
    "        self.head = create_head(head_inp_sz, 340, ps=0, lin_ftrs=[4000, 2000])[2:]\n",
    "        \n",
    "    def forward(self, drawings_lrg, drawings_small, seqs, country_codes):\n",
    "        x = torch.cat(\n",
    "            (\n",
    "                self.conv_lrg(drawings_lrg).squeeze(),\n",
    "                self.conv_small(drawings_small).squeeze(),\n",
    "                self.cnn_lstm(seqs.float()),\n",
    "                self.country_emb(country_codes)\n",
    "            ),\n",
    "            -1\n",
    "        )\n",
    "        return self.head(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data_bunch, MixedInputModel(), metrics=[accuracy, map3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.model = nn.DataParallel(learn.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEKCAYAAADuEgmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8VOW9x/HPb7KShbAlyCogmyCyBXGlUluXurfW1ltbxVqrtXrb3nptr9bui622blVKbdVW7eLaaltEaxVQQcMiILuAbErCngSyzu/+MQPGGMhAcjJnku/79ZoXM+ecmfk9zCTfnPOc8zzm7oiIiCQikuwCREQkdSg0REQkYQoNERFJmEJDREQSptAQEZGEKTRERCRhCg0REUmYQkNERBKm0BARkYSlJ7uAQ9WjRw8fMGBAsssQEUkp8+bN2+ruhS19nZQLjQEDBlBSUpLsMkREUoqZvdMar6PDUyIikjCFhoiIJEyhISIiCVNoiIhIwhQaIiKSMIWGiIgkTKEhIiIJU2iIiKSAO15YycyVZckuQ6EhIhJ27s7dL65m7tptyS5FoSEiEnZ7a+upjzr52RnJLkWhISISduVVdQDkZSV/5CeFhohIyO0LjfxshYaIiDSjolqhISIiCSqvqgUgL0t9GiIi0oyKjnJ4ysy6mNnjZrbczJaZ2QkH2G6CmdWb2UVB1iMikorC1BEedAV3AtPd/SIzywRyGm9gZmnArcBzAdciIpKSyjtCn4aZdQYmAb8DcPcad9/ZxKbXAU8ApUHVIiKSyipCtKcR5OGpQUAZ8ICZLTCz+80st+EGZtYHuBCYGmAdIiIprbyqlk4ZaaSnJb8bOsgK0oFxwH3uPhaoBL7VaJs7gBvdvf5gL2RmV5lZiZmVlJUlf+wVEZG2VFFdF4pDUxBsaGwENrr73Pjjx4mFSEPFwJ/NbB1wEXCvmV3Q+IXcfZq7F7t7cWFhYYAli4iET3lVHXkhCY3AqnD398xsg5kNc/cVwGnA0kbbDNx338weBJ5196eDqklEJBWVV9eRH4L+DAj+7KnrgEfiZ06tAaaY2dUA7q5+DBGRBFRU1YZisEIIODTcfSGxQ1ANNRkW7n55kLWIiKSq8qo6ivKzk10GoCvCRURCr6N0hIuISCsIU0e4QkNEJMSiUY/taYSkI1yhISISYpU1+4YQCUdHuEJDRCTE9g9WqMNTIiLSnDBNwAQKDRGRUHt/AiaFhoiINCNM84ODQkNEJNTePzyljnAREWlGmGbtA4WGiEiohWl+cFBoiIiEWnl1HWaQm6nQEBGRZpRX1ZKXmU4kYskuBVBoiIiEWkWIxp0ChYaISKiVV9WFphMcFBoiIqEWpmHRQaEhIhJq5dV15IXkGg1QaIiIhFp5Va32NEREJDEVVeGZSwMUGiIioaaOcBERSUhdfZS9tfWhGXcKFBoiIqFVWV0PhGcCJlBoiIiE1u74XBrqCBcRkWbtHxZdfRoiItKcsM0PDgoNEZHQqqjed3hKHeEiItKMsE3ABAoNEZHQ2hcanTvK4Skz62Jmj5vZcjNbZmYnNFr/OTNbFL+9amajg6xHRCSV7OsID1OfRtCV3AlMd/eLzCwTyGm0fi3wEXffYWZnAdOAiQHXJCKSEsqrakmLGJ0y0pJdyn6BhYaZdQYmAZcDuHsNUNNwG3d/tcHDOUDfoOoREUk1FfEhRMzCMWsfBHt4ahBQBjxgZgvM7H4zyz3I9l8E/tXUCjO7ysxKzKykrKwsiFpFREKnvDpc405BsKGRDowD7nP3sUAl8K2mNjSzycRC48am1rv7NHcvdvfiwsLCoOoVEQmV8qpwTcAEwYbGRmCju8+NP36cWIh8gJkdC9wPnO/u2wKsR0QkpVR0pNBw9/eADWY2LL7oNGBpw23MrD/wJPB5d18ZVC0iIqmovLo2dIengq7mOuCR+JlTa4ApZnY1gLtPBW4BugP3xjt66ty9OOCaRERSQkVVHYN65CW7jA8INDTcfSHQOASmNlh/JXBlkDWIiKSqiuq6UF2jAboiXEQktHZ3pD4NERE5fNV19dTURUM1LDooNEREQqkihIMVgkJDRCSU9k/AFKJh0UGhISISSmGcgAkUGiIiobQvNNQRLiIizXp/fnAdnhIRkWaUV8WmetXhKRERadb7HeEKDRERaUYY5wcHhYaISCiVV9WRmRYhO0Sz9oFCQ0QklCqqa0PXnwEKDRGRUCqvCt+sfaDQEBEJpTBOwAQKDRGRUArj/OCg0BARCaXY/ODhurAPFBoiIqFUUV2rw1MiIpKYcvVpiIhIItydCp09JSIiidhTU09d1NWnISIizSsrrwagMD8ryZV8mEJDRCRkyipioVGk0BARkeZoT0NERBJWursK0J6GiIgkoKyimvSI0TUnM9mlfIhCQ0QkZEp3V9MjL4tIxJJdyocoNEREQqasojqU/RkQcGiYWRcze9zMlpvZMjM7odF6M7O7zGy1mS0ys3FB1iMikgpKd3fQ0ADuBKa7+3BgNLCs0fqzgCHx21XAfQHXIyISemUV1aHsBIcAQ8PMOgOTgN8BuHuNu+9stNn5wB88Zg7Qxcx6BVWTiEjY1UedbR308NQgoAx4wMwWmNn9ZpbbaJs+wIYGjzfGl4mIdEjbKquJejhPt4VgQyMdGAfc5+5jgUrgW422aerUAG+8wMyuMrMSMyspKytr/UpFREIizBf2QbChsRHY6O5z448fJxYijbfp1+BxX2Bz4xdy92nuXuzuxYWFhYEUKyISBqX7QyM7yZU0LbDQcPf3gA1mNiy+6DRgaaPN/g58IX4W1fHALnd/N6iaRETCbt+eRlgPTwU9WPt1wCNmlgmsAaaY2dUA7j4V+CfwCWA1sAeYEnA9IiKhFvbDU4GGhrsvBIobLZ7aYL0D1wZZg4hIKikrryY/O53sjLRkl9IkXREuIhIiZeXhPd0WEgwNMzvKzLLi9081s+vNrEuwpYmIdDyl5VWh7c+AxPc0ngDqzWwwsYv1BgKPBlaViEgHFdvTCOeZU5B4aETdvQ64ELjD3b8O6MptEZFWVloe3iFEIPHQqDWzS4DLgGfjy8I347mISAqrrK5jT019qPs0Ej17agpwNfBjd19rZgOBh4MrK3g799SwccdeuudlUpSfTVqjcevro055VS2Z6RE6ZaRh1rrj2rs7lTX11NVHAbD4xfGdMtPITNf5CSId0f4L+/JSPDTcfSlwPYCZdQXy3f1nQRbW2t7avIu/vrGBVaUVrCqt2H8uNEB6xOjZOZuizllUVNWxvbKGHXtqiMYHNDGD3Mx0cjJjp8DVR526qBONOmlpRmZahMz0yP6AyclMo1NmOjkZadRFo+ypqY/f6qisrmd3VS2V1XX7X7+xgk4Z9MjLpEdeFgWdMsjOSCM7I0J2RhoRM2rro9TWR6mpi1IbderrY/XUR6PUeyyQou64x2qN+r71Tl39++ui7phBp8x0cjPTyIm3MWLsD0kzyMlMIzcrff//QX3U979/XdQpzM+iT5dO9O2aQ5+uneicnd7qISvSEey/sK9zioeGmb0EnBfffiFQZmYvu/s3AqytVb23q4rH521kcM98Th1ayJCeefTrmsO2yhre3bWXzTurKC2vomd+NhMGZtIjN5OCnExq66NUVsd+2e+pqcMM0iJGeiRCxIy6aOyXZ01dlOq6KFW1sYDYtbeW93btJS0SISczjfzsdHp2ziIvK4P87HTys9PJy0onMz2Cx8PDgT3VdZRVVLO1opqt5TWs376Hqtp6qmqjVNXVUx91stIjZKTFbulpRkYkEqspzTAz0uK/9Pf98s9Ii5CdYaRHjLSIEbH4LQLRKOytraeyuo4de/ayt+b9MHP8A+ur66If+n9Nixj1jdIvLWL725iflUGXnAy65WbSPTeTbrlZ9OqSTf9uOfTvlsMRnbNDOTuZSDKE/cI+SPzwVIG77zazK4EH3P27ZrYoyMJa26nDiljy/TP0F3AL1NVH2VNbT3okFkTp8V/22ytjh/o27dzLph172bm3hvKqOsqr6ti9t5Yde2p4a/NutlVUs7uq7gOvmZkW4YiCbIrysyjqnEVRfixQhh2Rz9Ce+fTIy9RnJh1GaXkVAEUhPnsq0dBIj89zcTFwU4D1BKZxn4UcuvS0CJ3TPtzf0j0vi+55WYzu1/ylOzV1Ud7dtZf12/fwzrY9bNi+h3d3xfbylr9XzsyVW6mofj9YuuZkMLRn/v4QGXZE7NY5W+dhSPtTVl5NesTo0im83+9EQ+MHwHPAK+7+hpkNAlYFV5a0V5npEY7snsuR3XM5ZciH17s7WytqWLmlnJVbylnxXuzfJ+dv+kCYDOqRy6i+BRzbtwuj+xZwTJ+C0A67IJKo0vjV4GE+ZJtoR/hjwGMNHq8BPhVUUdJxmRmF+VkU5mdx0uAe+5e7O5t3VbHyvXLe2ryLRRt3MXfNdv62MDaSfkaaMaJ3AWP7dWFs/y6M7tuFI7vn6NCWpJSwDyECiXeE9wXuBk4i1l87G/hvd98YYG0i+5kZfbp0ok+XTkweXrR/eenuKhZu2Mn89TuZv34Hf35jPQ++ug6AztnpjOpbwNh+XZk8vIgx/broMKWEWll5Nb0KwtufAYkfnnqA2LAhn44/vjS+7ONBFCWSqKLO2Zw+8ghOH3kEALX1UVZuKWfxxl0s2rSLxRt3cd/Lb3PPf1bTPTeTU4cV8fERPfno8CJdDyOhU1pezeh+Bcku46ASDY1Cd3+gweMHzexrQRQk0hIZaRFG9i5gZO8CPhtftmtPLS+tLOXF5aW8sGwLT8zfSLfcTC4Y04eLJ/Rl+BGdk1qzCMSuqdpeWR3qC/sg8dDYamaXAn+KP74E2BZMSSKtqyAng/PH9OH8MX2oq48ya/VWHi/ZyB/nrOP3r6xldN8CPjOhP+eN6U1eVtDzkok0bVtFNVGHws7t4/DUFcA9wK+I9Wm8imbZkxSUnhZh8rAiJg8rYntlDU8v2MRf3tjA/z21mB/9YynnHtubSyb2Z0wCpw+LtKZUGEIEEj97aj2xK8L3ix+euiOIokTaQrfcTK44eSBTThrAwg07+fPrG3hm0Wb+UrKB4wZ249rJg5k0pIfOwJI2kQpDiEDLZu5LmSFERA7GzBjbvyu3XnQsc//vNG45ZwQbtu/hst+/zrn3zOZfi98leqCBwkRaSVmK7Gm0JDT055e0O/nZGVxx8kBevmEyP//UsVRW13PNI/O58L5XKVm3PdnlSTtWVhH+caegZaGhP72k3cpMj3DxhH688I2PcNunR7NlVxUXTX2Nax6exzvbKpNdnrRDpbur6JydHvqRDQ7ap2Fm5TQdDgZ0CqQikRBJixgXje/L2aN68dtZa5j68tu8sGwLl50wgOtOG0JBiMcIktRSVhH+q8GhmT0Nd893985N3PLdXecmSofRKTON608bwkvfPJVPju3L715Zy+TbXuKPr63bP5GWSEuU7q4O9ei2++iSWJFDUNQ5m1svOpZnrzuZYT3z+c7f3uKsO2cxd40uW5KWaRd7GiLStJG9C3j0SxOZ9vnxVNXV85lpc7jlb0s+MBKvSKLcPb6nodAQabfMjNNHHsFzX5vEFScN5I9z3uGMX81k1qqyZJcmKaaiuo69tfXa0xDpCHIy07nl3BE8fvUJZGVE+PzvXuc7Ty+hqrY+2aVJilhVWgHAkd1zk1xJ8xQaIq1k/JHd+Of1p3DlybG9jvPveYWVW8qTXZakgCWbdgEwqm+4R7iFgEPDzNaZ2WIzW2hmJU2sLzCzZ8zsTTN7y8w0npWktOyMNG4+ZwQPTpnAtspqzr17Ng/PeQd3XdYkB7Z44y665WbSO+RzaUDb7GlMdvcx7l7cxLprgaXuPho4FbjdzDLboCaRQJ06rIh//vcpHDewGzc/vYQv/3EeOyprkl2WhNTiTbs4pk9BSoxzluzDUw7kW+x/Kg/YDuj0E2kXivKzeWjKcdz0iaP5z4pSzrxzJrNXbU12WRIyVbX1rCqt4Ng+4T80BcGHhgMzzGyemV3VxPp7gKOBzcBiYlPIfuhKKTO7ysxKzKykrExnpkjqiESML00axNPXnkReVjqX/m4uP/7HUqrr1EkuMUvf3U191DlGoQHASe4+DjgLuNbMJjVafwawEOgNjAHuMbMPTaPm7tPcvdjdiwsLCwMuWaT1jexdwLPXncKlx/fnt7PWcsm0OZSWVyW7LAmBVOoEh4BDw903x/8tBZ4Cjmu0yRTgSY9ZDawFhgdZk0iydMpM40cXjOLez41j2bvlnH/PK/t/YUjHlUqd4BBgaJhZrpnl77sPnA4sabTZeuC0+DY9gWHAmqBqEgmDT4zqxePXnIABF019lWcXbU52SZJEqdQJDsHuafQEZpvZm8DrwD/cfbqZXW1mV8e3+SFwopktBv4N3Oju6imUdm9k7wL+9tWTOaZ3AV99dAG/nLFCEz11QKnWCQ6JzxF+yNx9DTC6ieVTG9zfTGwPRKTDKczP4pEvTeQ7Ty/hrhdXs2JLOb+8eAy5WRpAuqNYlmKd4JD8U25FOrSs9DRu/dSx3HLOCJ5fuoVP3fcqG7bvSXZZ0kYWp1gnOCg0RJLOzLji5IE8OOU4Nu/cy/m/foU3NLVsh5BqneCg0BAJjUlDC/nbV0+mS04Gl94/l/+sKE12SRKwVOsEB4WGSKgM7JHLY18+gSE98/jSQyU6s6odS8VOcFBoiIRO97wsHv3S8Yzr35Xr/7SAv7yxPtklSQBSsRMcFBoiodQ5O4OHrjiOU4YUcuMTi/n97LXJLklaWapdCb6PQkMkpDplpvHbLxRz5sgj+MGzS3l0rvY42pNFKdgJDgoNkVDLTI9w1yVjmTyskJueXszTCzYluyRpJanYCQ4KDZHQy0yPcN+l45k4sBv/89ibTF/yXrJLkhZK1U5wUGiIpITsjDTuv2wCo/oUcP2fFvDySk0RkMoWbthJfdQZ3a9Lsks5ZAoNkRSRl5XOQ1OO46iiPL70hxIFRwqbvWoraRFj4qBuyS7lkCk0RFJIQU4Gj1w5kcGFseDQBYCpadbqrYzp14XO2RnJLuWQKTREUky33EweuXIiQ4ry+PIf5vGf5QqOVLJrTy2LN+7k5ME9kl3KYVFoiKSgrvHgGHZEPl/+4zz+vWxLskuSBL369laiDqcMUWiISBvqkpPJw1+cyPBe+Xzlkfks3LAz2SVJAmat3kpeVnpKdoKDQkMkpRXkZPDA5RMo6pzFlQ+VsHGHhlUPu1mryjh+UHcy0lLz129qVi0i+3XPy+KByydQXVfPFQ++we6q2mSXJAfwzrZKNmzfm7KHpkChIdIuDC7K5zeXjmdNWSXXPjKf2vposkuSJsxaFZvN+mSFhogk24mDe/CTC0cxa9VWvv/MW8kuR5owe9VWehdkM6hHbrJLOWwKDZF25OIJ/fjyRwbx8Jz1PFayIdnlSAP1UefVt7dy8pAeKTfeVEMKDZF25obTh3HiUd25+eklvLV5V7LLkbhFG3eyu6qOk4cUJruUFlFoiLQz6WmxkXG75GRwzcPz2bVXHeNhMDven3HSUd2TXEnLKDRE2qEeeVnc+7lxbN65l//560KiUU92SR3erNVbGdm7M93zspJdSosoNETaqfFHduPms4/mhWWl3Pfy28kup0OrrK5jwfodKX3W1D4KDZF27LITB3Du6N7cPmMFc9dsS3Y5HdaTCzZRW++cNrxnsktpMYWGSDtmZvz0k6M4snsu1/95AdsqqpNdUodTVVvPr19cTfGRXZkwoGuyy2kxhYZIO5eXlc49/zWWHXtq+fpf31T/Rhv78+vreW93Fd/4+NCUPtV2n0BDw8zWmdliM1toZiUH2ObU+Pq3zOzlIOsR6ahG9i7glnNGMHNlGVNnqn+jrVTV1vPrl95m4sBunJDiZ03tk94G7zHZ3bc2tcLMugD3Ame6+3ozK2qDekQ6pM9N7M9ra7Zx+4yVTBjQjQkDUm/WuFTz8Jx3KCuv5u5LxraLvQxI/uGp/wKedPf1AO6u2WREAmJm/OyTo+jbtRPX/2kBu/bo+o0g7ampY+rLb3PS4O4cP6h97GVA8KHhwAwzm2dmVzWxfijQ1cxeim/zhYDrEenQ8rMzuPuSsZSWV/Pdvy9Jdjnt2sNz3mFrRQ1f/9jQZJfSqoIOjZPcfRxwFnCtmU1qtD4dGA+cDZwBfMfMPvQ/bGZXmVmJmZWUlZUFXLJI+3Zs3y5c99HBPL1wM/9Y9G6yy2mXKqvrmPryGk4Z0oPidnYYMNDQcPfN8X9LgaeA4xptshGY7u6V8X6PmcDoJl5nmrsXu3txYWFqj9siEgbXTh7M6L4F3PT0Ykp3VyW7nHbnjhdWsr2yhm98vH3tZUCAoWFmuWaWv+8+cDrQeH/4b8ApZpZuZjnARGBZUDWJSExGWoTbLx7D3pp6/veJRbjrNNzW8uaGnfxu9lo+N7E/Y/un/nUZjQW5p9ETmG1mbwKvA/9w9+lmdrWZXQ3g7suA6cCi+Db3u7sOtIq0gcFFeXz7rOG8tKKMR19fn+xy2oXa+ig3PrGIwvwsbjxreLLLCURgp9y6+xqaPtQ0tdHjXwC/CKoOETmwL5wwgBeWlfKjZ5dxyuBC+nfPSXZJKW3azDUsf6+caZ8fT+fsjGSXE4hkn3IrIkkUiRg/v+hY0iPGDY/ravGWeLusgjv/vYqzR/Xi9JFHJLucwCg0RDq43l06cfM5RzN37XYenvtOsstJSdGo8+0nF5OdHuG7541IdjmBUmiICBcX92PS0EJ++s/lrN+2J9nlpBR35+fPreD1tdu5+ewRFOVnJ7ukQCk0RGT/1eI6THXofvX8Sqa+/Db/NbE/ny7um+xyAqfQEBFAh6kOx13/XsVdL67mM8X9+NH5x7Sb8aUORqEhIvs1PEy1aefeZJcTave+tJpfPr+ST47rw08/OYpIpP0HBig0RKQBM+MnFx5D1J2f/FPX2TalsrqOm59ezM+nr+D8Mb35xUWjO0xggEJDRBrp2zWHa049in8sepdX325yVoMOa86abZx550wembueL548kNs/PZq0DhQYoNAQkSZc/ZGj6Nu1E9//+1Lq6qPJLifpSsur+P4zb/HZaXOImPGXq07gO+eMID2t4/0KbYtJmEQkxWRnpHHz2SO4+uF5PDznHS4/aWCyS2pT1XX1vLJ6K7NXbWP26jJWbqkA4LITjuTGs4aTk9lxf3V23JaLyEGdMbInJw/uwS+fX8m5o3vTPS8r2SUFrq4+ypMLNnHnC6vYtHMvWekRjhvYjU+O68upwwoZfkTnZJeYdAoNEWmSmfG980Zw5h2zuG3GCn76yWOTXVJgolHnX0ve4/bnV7CmrJJj+xbw/fNGcvKQHmRnpCW7vFBRaIjIAQ0uyufyEwfwu1fW8vnjBzCid/v6S7u2Psozb27mNy+vYcWWcoYU5TH10vGcMbJnh7jm4nAoNETkoK47bQiPzdvIz59bzoNTGs+jlpr21NTxlzc2cP+stWzauZdhPfO54zNjOHd07w53NtShUmiIyEEVdMrgK6cexU//tZw5a7Zx/KDuyS7psJWVV/OH19bxxznvsHNPLRMGdOWHF4xk8rAi7VkkSKEhIs267MQBPPDKOm6dvpwnrzkxpX7BRqPOW5t38+jr63li/kZq66N8/OieXDVpULubv7stKDREpFnZGWl87WND+NaTi5mxdAtnhHy+iB2VNcxcVcbLK8qYuaqMrRU1ZKZH+NS4vlx5ykCOKsxLdokpS6EhIgm5aHxffjtrDb94bgWnDS8K3YVt7+7ay4y3tjB9yXvMXbuNqEPXnAwmDS3k1GGFfGRoEd1yM5NdZspTaIhIQtLTItxwxjCufng+Ty7YxMXF/ZJaT01dlAXrdzBzVRkzV25l8aZdQGzu86+cOpiPjejJqD4F6thuZQoNEUnYGSOPYHS/Ltzx/ErOG907KdcwrCmr4K5/r+L5pVuorKknLWKM69+FG84Yxhkjj2BwkQ49BUmhISIJMzO+fdZwPjttDne/uIobzhjeZu/93q4q7vz3Kv5asoGs9AgXjO3DR4YWcuJR3cnPzmizOjo6hYaIHJLjB3Xn0+P78puX13D2qN6BX/BXXlXLr//zNg+8spaoO58//kiunTyYwvz2P6xJGCk0ROSQ3XT20fxnRSnfenIRT15zYiCd4vVR57GSDdw2YwVbK2q4cGwfvvHxofTrltPq7yWJC9fpDyKSErrkZPK980ayaOMuHnx1Xau//utrt3Pu3bP51pOLGdA9l79/9SR+9ZkxCowQ0J6GiByWs0f14umjN3HbjBWcPuII+ndvnV/oT87fyDcfe5NeBZ24+5KxnHNsr5S6mLC9056GiBwWM+OHFxxDeiTC/z21GHdv8Ws+Mvcd/uexN5k4sDvPfX0S547urcAIGYWGiBy2XgWduPGs4cxevZV7Xlzdote6f9YabnpqCZOHFfHAlAnkZelASBjpUxGRFvnccf2Z/84Obn9+JT07Z3PxhEO/6O+eF1dx24yVnD2qF7/6zBgy0/X3bFgF+smY2TozW2xmC82s5CDbTTCzejO7KMh6RKT1RSLGrZ86llOG9ODbTy3mxeVbDun5C9bv4LYZK7lwbB/u/KwCI+za4tOZ7O5j3L24qZVmlgbcCjzXBrWISAAy0yPcd+l4ju6Vz7WPLGDhhp0JPc/duXX6cnrkZfKjC44J3XhW8mFh+ISuA54ASpNdiIgcvrysdH5/+QR65GdyxYNvsG5rZbPPeXllGXPWbOe6jw4hV30YKSHo0HBghpnNM7OrGq80sz7AhcDUg72ImV1lZiVmVlJWVhZQqSLSUkX52fzhiom4O1c89Aa79tQecNto1Ll1+gr6d8vhkuP6t2GV0hJBh8ZJ7j4OOAu41swmNVp/B3Cju9cf7EXcfZq7F7t7cWFhYVC1ikgrGNgjl6mXjmfD9j1c++h8auujTW73zKLNLHt3N/9z+lD1Y6SQQD8pd98c/7cUeApoPMFwMfBnM1sHXATca2YXBFmTiARv4qDu/PjCUcxevZXvP/PWh67hqKmLcvuMlYzo1Zlzj+2dpCrlcAR2ENHMcoGIu5fH758O/KDhNu4+sMH2DwLPuvvTQdUkIm3n4uJ+vF2hu6WSAAAJAklEQVRawW9mrmFwYR6Xn7T/x50/vb6e9dv38OCUCUQ030VKCbLnqSfwVPxqznTgUXefbmZXA7j7QfsxRCT1/e+Zw3m7rJIfPLuUh157BwMweHdnFccP6sZHhupwc6qx1rj0vy0VFxd7SckBL/kQkZCprK7j9hkr2VpRTdQdByJmXPfRwQztmZ/s8joMM5t3oEsfDoXOcRORQOVmpXPLuSOSXYa0Ep2yICIiCVNoiIhIwhQaIiKSMIWGiIgkTKEhIiIJU2iIiEjCFBoiIpIwhYaIiCQs5a4IN7My4J1GiwuAXc0sO9jjffcbLusBbD3MMpuqJ9FtDrUtzd1vSTsOVmci68PUlpZ8Jk2t6yjfr8aPG7cl6O/XwbZpz9+vppa1tC1HunvLx21x95S/AdOaW3awx/vuN1pW0pr1JLrNobalufstaUcibTnY+jC1pSWfyaF+n9rT96u5tgT9/WrNtqTS9yuZbWnu1l4OTz2TwLKDPX7mANu0Zj2JbnOobUnkfks09zoHWx+mtrTkM2lqXUf5fjV+nMptSaXvV1PL2vLn/oBS7vBUWzGzEm+Fwb2Srb20A9SWMGov7QC1JVHtZU8jCNOSXUAraS/tALUljNpLO0BtSYj2NEREJGHa0xARkYS1+9Aws9+bWamZLTmM5443s8VmttrM7rL4NITxddeZ2Qoze8vMft66VR+wnlZvi5l9z8w2mdnC+O0TrV95k/UE8rnE13/TzNzMerRexQetJ4jP5Ydmtij+mcwws8An0g6oHb8ws+XxtjxlZl1av/Im6wmiLZ+O/7xHzSzQvo+W1H+A17vMzFbFb5c1WH7Qn6UmBXVaVlhuwCRgHLDkMJ77OnACYMC/gLPiyycDLwBZ8cdFKdyW7wHfbA+fS3xdP+A5Ytfy9EjVtgCdG2xzPTA1RdtxOpAev38rcGsKfyZHA8OAl4DiMNYfr21Ao2XdgDXxf7vG73c9WFsPdmv3exruPhPY3nCZmR1lZtPNbJ6ZzTKz4Y2fZ2a9iP3gvuax/90/ABfEV18D/Mzdq+PvURpsK2ICaktSBNiWXwH/C7RZZ10QbXH33Q02zaUN2hNQO2a4e1180zlA32BbERNQW5a5+4ow138AZwDPu/t2d98BPA+cebi/F9p9aBzANOA6dx8PfBO4t4lt+gAbGzzeGF8GMBQ4xczmmtnLZjYh0GoPrqVtAfhq/PDB782sa3ClNqtFbTGz84BN7v5m0IUmoMWfi5n92Mw2AJ8Dbgmw1oNpje/XPlcQ+2s2WVqzLcmQSP1N6QNsaPB4X5sOq60dbo5wM8sDTgQea3D4LqupTZtYtu+vvXRiu3nHAxOAv5rZoHhat5lWast9wA/jj38I3E7sh7tNtbQtZpYD3ETscEhStdLngrvfBNxkZt8Gvgp8t5VLPajWakf8tW4C6oBHWrPGRLVmW5LhYPWb2RTgv+PLBgP/NLMaYK27X8iB23RYbe1woUFs72qnu49puNDM0oB58Yd/J/bLtOGudF9gc/z+RuDJeEi8bmZRYmO9lAVZeBNa3BZ339Lgeb8Fng2y4INoaVuOAgYCb8Z/qPoC883sOHd/L+DaG2uN71hDjwL/oI1Dg1ZqR7zj9RzgtLb+w6qB1v5M2lqT9QO4+wPAAwBm9hJwubuva7DJRuDUBo/7Euv72MjhtDXIzpyw3IABNOhQAl4FPh2/b8DoAzzvDWJ7E/s6iT4RX3418IP4/aHEdv0sRdvSq8E2Xwf+nKqfS6Nt1tFGHeEBfS5DGmxzHfB4irbjTGApUNhWn0XQ3y/aoCP8cOvnwB3ha4kdHekav98tkbY2WVdbf5BJ+OL8CXgXqCWWrF8k9hfpdODN+Bf6lgM8txhYArwN3MP7F0NmAg/H180HPprCbfkjsBhYROwvrV6p2pZG26yj7c6eCuJzeSK+fBGx8YT6pGg7VhP7o2ph/Bb4WWABtuXC+GtVA1uA58JWP02ERnz5FfHPYjUw5VB+lhrfdEW4iIgkrKOePSUiIodBoSEiIglTaIiISMIUGiIikjCFhoiIJEyhIe2CmVW08fvdb2YjWum16i02mu0SM3umuZFgzayLmX2lNd5b5FDplFtpF8yswt3zWvH10v39gfYC1bB2M3sIWOnuPz7I9gOAZ939mLaoT6Qh7WlIu2VmhWb2hJm9Eb+dFF9+nJm9amYL4v8Oiy+/3MweM7NngBlmdqqZvWRmj1tsTohH9s03EF9eHL9fER9c8E0zm2NmPePLj4o/fsPMfpDg3tBrvD8AY56Z/dvM5ltszoPz49v8DDgqvnfyi/i2N8TfZ5GZfb8V/xtFPkChIe3ZncCv3H0C8Cng/vjy5cAkdx9LbPTYnzR4zgnAZe7+0fjjscDXgBHAIOCkJt4nF5jj7qOBmcCXGrz/nfH3b3ZMn/g4SKcRuzIfoAq40N3HEZvD5fZ4aH0LeNvdx7j7DWZ2OjAEOA4YA4w3s0nNvZ/I4eiIAxZKx/ExYESDUUE7m1k+UAA8ZGZDiI3qmdHgOc+7e8N5DF53940AZraQ2HhAsxu9Tw3vD/Q4D/h4/P4JvD8/waPAbQeos1OD155HbL4DiI0H9JN4AESJ7YH0bOL5p8dvC+KP84iFyMwDvJ/IYVNoSHsWAU5w970NF5rZ3cB/3P3CeP/ASw1WVzZ6jeoG9+tp+mem1t/vHDzQNgez193HmFkBsfC5FriL2DwahcB4d681s3VAdhPPN+Cn7v6bQ3xfkUOmw1PSns0gNg8FAGa2b1jpAmBT/P7lAb7/HGKHxQA+29zG7r6L2NSu3zSzDGJ1lsYDYzJwZHzTciC/wVOfA66Iz7mAmfUxs6JWaoPIByg0pL3IMbONDW7fIPYLuDjeObyU2JD2AD8HfmpmrwBpAdb0NeAbZvY60AvY1dwT3H0BsVFMP0tswqJiMyshttexPL7NNuCV+Cm6v3D3GcQOf71mZouBx/lgqIi0Gp1yKxKQ+GyCe93dzeyzwCXufn5zzxMJM/VpiARnPHBP/IynnSRhGl2R1qY9DRERSZj6NEREJGEKDRERSZhCQ0REEqbQEBGRhCk0REQkYQoNERFJ2P8DVIexNJAFeFgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "... did some training here ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replacing the classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Freezing your arch can be very helpful for trying out a new classifier (head)! As it turned out, with a bigger classifer, I jumped 100 spots on the public LB!\n",
    "\n",
    "(cnn_lstm etc are submodules of module which is added through DataParallel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict = torch.load('models/resnet50-incv4-2e-05.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = list(state_dict.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in keys:\n",
    "    if 'module.head' in k: del state_dict[k]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At this point I modify the architecture above and rerun the cells. I have the state_dict ready so all I need is to load it.\n",
    "\n",
    "(the weights in the lower level of the model have been retained - I will need to train the classifier from scratch, it has been reinitialized with random weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.model.load_state_dict(state_dict, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Freezing submodules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in [learn.model.module.cnn_lstm, learn.model.module.conv_lrg, learn.model.module.conv_small]:\n",
    "    requires_grad(m, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8VNX9//HXZ7KSFQIBwhqUXRSEQFVa12oFqWjdrXX91drWWvVrW3dbtVpb22rV1rrUWq3VVqvivq9VWSK7IIIkEEASEgghIfv5/TFDjCEbJHfuzOT9fDzmwcydMzOf48S8c++59xxzziEiIgIQ8LsAERGJHAoFERFpolAQEZEmCgUREWmiUBARkSYKBRERaeJpKJhZbzN70sxWmtkKMzu4xfNmZn8ys9VmtsTMJntZj4iItC/e4/e/E3jZOXeymSUCKS2enwGMCt2+Bvwl9K+IiPjAsz0FM8sADgUeBHDO1TrntrVoNhv4hwv6COhtZjle1SQiIu3zck9hH6AEeMjMJgL5wE+dc5XN2gwG1jd7XBTatqmtN+3Xr5/Lzc3t/mpFRGJYfn7+FudcdkftvAyFeGAy8BPn3FwzuxO4EriuWRtr5XW7zbthZhcCFwIMGzaMBQsWeFCuiEjsMrPCzrTzcqC5CChyzs0NPX6SYEi0bDO02eMhwMaWb+Scu885l+ecy8vO7jDoRERkL3kWCs65L4D1ZjYmtOko4JMWzeYAZ4fOQjoIKHfOtXnoSEREvOX12Uc/Af4ZOvPoc+A8M7sIwDl3L/AiMBNYDVQB53lcj4iItMPTUHDOLQLyWmy+t9nzDvixlzWIiEjn6YpmERFpolAQEZEmCgUREWni9UBzxFi1uYLnF28kNSme1KR40kL/xscZcWYEzAgECN4PhB4bxAWM+ECAhDgjPi5AfGDX88G21qzdrltiXID4OOWtiESfHhMKn23ewV1vrSZcS1KnJsaR0SuBzF4JpCbF45yjodHR4BwNjdDY6Gh0wZtzwVBJSYonNTGOlMR4UpPiSE2KJ/0rIfblcymJ8WSlJpKdlkTvlATMWrsOUERkz/SYUDjugBxmTJjJzroGKmvqqaipp6qmgbrGxtAvbGhodMH7ztHogr+4Gxod9Y2N1DV8+a/b9bxzoV/uwdfu+qVfXddARXU95Tvr2L6zjsra+tCeR3BPIvgvob2T4OP6hkaqahuoqq1n47adVNbWB+usrqemvrHdviXEGf3SkshKTSQ9OZ60pATSk+NJT44nIzl4P6NXAhnJCfRNSyQ7PYns9CTSk+IVJiLyFT0mFAACAWs6fNTf72L2QF1DIzuq66mqa6Cqpp7K2mCwba2qpXh7DSU7aiipqKGsspYd1fVs2LaTiuo6Kqrrqaiuo7GNvaPkhADZ6UkMSE9mQEYy/TOSyMlMZnjfVEb0S2VYVgrJCXHh7ayI+KpHhUK0SogL0Cc1kT578VrnHJW1DVRU11G+s47SHbWUVARDpLiimpKKGjZvr2HFF9t5Z1UNO2rqm15rBoMye5HbL4XcUFDk9k1lZP80hmWlEAhoL0Mk1igUYpyZkRYak8jJ7NVh+/KqOgpKKykorWTtlkoKtlSytrSK5xZvZHv1l4HRKyGOUQPSGD0gnbED05kwOJPxgzLISE7wsjsi4jGFgnxFZkoCE1N6M3Fo769sd86xtaqOtVsqWV1cwadf7GDV5gre/rSEJ/OLmtrl9k1hv8GZ5A3vw9TcLMblZBCnPQqRqKFQkE4xM7JSE8lKTWTK8K8eyCqpqGH5xnKWb9zOsg3lLFq3jReWBOc1TEuKZ/LwPhw6qh9Hjx/A8L6pfpQvIp1kLlznaHaTvLw8p/UUIt/GbTuZX1DGvLVlzF1bxuriHQCMHpDGN8cN4NgJA9l/cKbOfhIJEzPLd861nItu93YKBQmHdaVVvL5iM6+v2MzctWU0NDr2yU7lxEmDOeHAwQzNarl8t4h0J4WCRKzyqjpeXr6Jpxdu4KPPywDIG96HU/OGctwBOaQm6aimSHdTKEhU2LBtJ88u2sCT+UV8XlJJSmIcsw7I4bSpQ5k8rI8OL4l0E4WCRBXnHPmFW/n3gvU8v2QTVbUNTBram4uPGMlR4/orHES6SKEgUWtHTT1PL9zAX99ZQ9HWnYwdmM7FR45kxoQcnd4qspcUChL16hoambNoI39+ezVrSioZMyCdq48bx2Gjs/0uTSTqdDYUNL+zRKyEuAAnTRnCq5cdxt1nHkh1fQPn/G0e33twLis2bfe7PJGYpFCQiBcXMGYdMIjXLjuM62aNZ0lROTP/9B6/eHIJpTtq/C5PJKYoFCRqJMYHuODrI3j3Z0dwwfQRPPVxEUfc/jYPf1BAfUP704uLSOcoFCTqZKYkcO2s8bx86aEcMKQ3N8xZzrfv/h8LCsr8Lk0k6ikUJGqN7J/GIxdM48/fnUx5VS0n3/shNzy7jKra+o5fLCKtUihIVDMzZu6fw+v/dxjnTx/Bwx8WMvPO98gv1F6DyN5QKEhMSEmM5/pvj+df3z+I+kbHKfd+yK0vraC6rsHv0kSiikJBYsrB+/bl5UsP5bSpQ/nrO59z2l8/ZFP5Tr/LEokaCgWJOWlJ8dz6nQO496wprC7ewbfv+h/zNQgt0ikKBYlZx04YyDM/nk5aUhxn3PcRj35U6HdJIhFPoSAxbdSAdJ69+Ot8fVQ/rn1mGdc8vVTXNIi0Q6EgMS+zVwIPnjOVHxy2D/+cu46LHv2YnbUagBZpjUJBeoS4gHHVjHHcOHs/3li5mbMenMu2qlq/yxKJOAoF6VHOPjiXe86czNKick6+90M2bNOZSSLNKRSkx5m5fw7/uGAam7dXc9KfP2BdaZXfJYlEDE9DwcwKzGypmS0ys90WQTCzw82sPPT8IjO73st6RHY5aJ++/PsHB1Nd38BZD86leHu13yWJRIRw7Ckc4Zyb1M7iDu+Fnp/knLsxDPWIADAuJ4O/nzeN0h01fO/BeRpjEEGHj6SHmzS0N/efncfaLZWc9/f5VNZoMj3p2bwOBQe8amb5ZnZhG20ONrPFZvaSme3XWgMzu9DMFpjZgpKSEu+qlR7pkJH9uOvMA1lSVM5Fj+ZTU6/TVaXn8joUpjvnJgMzgB+b2aEtnv8YGO6cmwjcBTzT2ps45+5zzuU55/Kys7U+r3S/b+03kN+edADvfbaFa55eRrStXS7SXTwNBefcxtC/xcDTwLQWz293zu0I3X8RSDCzfl7WJNKWk6YM4adHjeLJ/CIe+l+B3+WI+MKzUDCzVDNL33UfOAZY1qLNQDOz0P1poXpKvapJpCM/PWoU39pvAL9+cQXvf7bF73JEws7LPYUBwPtmthiYB7zgnHvZzC4ys4tCbU4GloXa/Ak43Wm/XXwUCBh/OHUSI7PT+PFjH1OwpdLvkkTCyqLtd3BeXp5bsGC3Sx5EutW60iqOv+d9stOS+O+PDiE9OcHvkkS6xMzy27k0oIlOSRVpxbC+Kfz5zMl8vqWSy/+9mMbG6PrjSWRvKRRE2nDIyH5cPXMcr32ymb+8s8bvckTCQqEg0o7zp+dy/MRB3P7qp7y7StfISOxTKIi0w8z4zUn7M2ZAOpc8vpD1ZZo8T2KbQkGkAymJ8dx71hQaGh0XPZpPdZ2ueJbYpVAQ6YTcfqnccdoklm/cznXPLOv4BSJRSqEg0klHjRvAJUeO5D/5RcxZvNHvckQ8oVAQ2QOXHDWKA4f15pqnl2rVNolJCgWRPRAfF+CO0ybR2Oi4/IlFNOj6BYkxCgWRPTS8byo3HL8fc9eWcf97n/tdjki3UiiI7IVTpgxhxoSB/P7VT1m2odzvckS6jUJBZC+YGbecuD9ZqYn89PGF7KzVaaoSGxQKInupT2oit58ykTUllfzx9VV+lyPSLRQKIl3wjVHZnDFtGA+89zkL1231uxyRLlMoiHTRVTPHMiAjmZ89uUTrO0vUUyiIdFFGcgK3fGd/Vhfv4K43VvtdjkiXKBREusERY/pz0uQh/OWdNTobSaKaQkGkm1w3axxZqYn87Mkl1NY3+l2OyF5RKIh0k94pidx8wgRWbNrOfe9qUR6JTgoFkW70rf0GMmPCQO56c7XWXpCopFAQ6WbXf3s8cQHjhjnLcU5zI0l0USiIdLOczF5c9s3RvLmymNc+2ex3OSJ7RKEg4oFzp+cyZkA6v3ruE6pq6/0uR6TTFAoiHkiIC3DziRPYsG0nf9K1CxJFFAoiHpmam8UpU4bwwHuf89nmCr/LEekUhYKIh66cMZbUpHiufWaZBp0lKigURDzUNy2JXxw7lrlry7Sus0QFhYKIx06bOpSJQzK5+YUVVFTX+V2OSLsUCiIeiwsYN86ewJYdNdzx+md+lyPSLoWCSBhMHNqbM6YN4+8fFLDyi+1+lyPSJoWCSJj87JgxZCTHc/0zutJZIpdCQSRM+qQm8otjxzKvoIynF27wuxyRVnkaCmZWYGZLzWyRmS1o5Xkzsz+Z2WozW2Jmk72sR8Rvp+YNZeLQ3tzy4grKd2rQWSJPOPYUjnDOTXLO5bXy3AxgVOh2IfCXMNQj4ptAwLh59gRKK2v542ur/C5HZDd+Hz6aDfzDBX0E9DazHJ9rEvHU/kMyOetrw/nHhwVapU0ijteh4IBXzSzfzC5s5fnBwPpmj4tC277CzC40swVmtqCkpMSjUkXC54pjxtAnJZHrnl1GY6MGnSVyeB0K051zkwkeJvqxmR3a4nlr5TW7/R/inLvPOZfnnMvLzs72ok6RsMpMSeCqmeNYuG4b/8lf3/ELRMLE01Bwzm0M/VsMPA1Ma9GkCBja7PEQQHMBSI9w0uTBTM3tw29eWsnWylq/yxEBPAwFM0s1s/Rd94FjgGUtms0Bzg6dhXQQUO6c2+RVTSKRxMy46YQJbK+u57evfOp3OSKAt3sKA4D3zWwxMA94wTn3spldZGYXhdq8CHwOrAbuB37kYT0iEWfswAzOPSSXx+evY9H6bX6XI4JF25WVeXl5bsGC3S55EIlaFdV1HPX7dxiYmcwzP5pOINDaUJtI15hZfhuXBnyF36ekivR46ckJXD1zHEuKynligQadxV8KBZEIMHvSIKblZvHbl1eyrUqDzuIfhYJIBDAzfjV7P7ZX13P7qxp0Fv8oFEQixLicDL530HD+OXedrnQW3ygURCLIZUePpm+qrnQW/ygURCJIZq8EfnHsWBau28aTHxf5XY70QAoFkQhz0uQhHDisN7e/8ilVtfV+lyM9jEJBJMIEAsa1x42juKKG+99d63c50sMoFEQi0JThWcyYMJC/vruG4opqv8uRHkShIBKhfn7sWGrrG/nja5/5XYr0IAoFkQg1ol8qZx00nCfmr+OzzRV+lyM9hEJBJIJdctQoUhPjufWllX6XIj1Ep0LBzPY1s6TQ/cPN7BIz6+1taSKSlZrIj44YyZsri/lgzRa/y5EeoLN7Ck8BDWY2EngQGAE85llVItLkvOm5DMpM5pYXV+iCNvFcZ0Oh0TlXD5wI3OGcuwzI8a4sEdklOSGOK741hmUbtvPcEi1MKN7qbCjUmdkZwDnA86FtCd6UJCItnTBpMONzMvjty59SXdfgdzkSwzobCucBBwO/ds6tNbMRwKPelSUizQUCxtUzx7Fh204e+bDQ73IkhnUqFJxznzjnLnHO/cvM+gDpzrnfeFybiDTz9VH9OGx0Nne9+ZnWXBDPdPbso7fNLMPMsoDFwENm9gdvSxORlq6cMZaKmnrueWu136VIjOrs4aNM59x24DvAQ865KcA3vStLRFozLieDkycP4eEPCllfVuV3ORKDOhsK8WaWA5zKlwPNIuKDy48ZjRlaoU080dlQuBF4BVjjnJtvZvsAmpBFxAc5mb244OsjeHbRRpYUbfO7HIkxnR1o/o9z7gDn3A9Djz93zp3kbWki0pYfHr4vfVMTufn5FTinC9qk+3R2oHmImT1tZsVmttnMnjKzIV4XJyKtS09O4PJjRjOvoIyXl33hdzkSQzp7+OghYA4wCBgMPBfaJiI+OS1vKKMHpHHrSyupqdcFbdI9OhsK2c65h5xz9aHb34FsD+sSkQ7ExwW49rjxrCur4uEPCvwuR2JEZ0Nhi5mdZWZxodtZQKmXhYlIxw4dnc3hY7K5643VlO6o8bsciQGdDYXzCZ6O+gWwCTiZ4NQXIuKza2aOo6qugTte1wmB0nWdPftonXPueOdctnOuv3PuBIIXsomIz0YNSOfMacN4bJ5WaJOu68rKa5d3WxUi0iWXHT2alMQ4fqMV2qSLuhIK1m1ViEiXZKUm8qPDR/LGymI+XKPhPtl7XQkFXTEjEkHOm55LTmYyt76kFdpk77UbCmZWYWbbW7lVELxmoUOhs5UWmtlucyaZ2blmVmJmi0K3/7eX/RDp8ZIT4vi/Y8awpKic55du8rsciVLthoJzLt05l9HKLd05F9/Jz/gpsKKd559wzk0K3R7odOUispsTDxzMuJwMfveKLmiTvdOVw0cdCk2FcRygX/YiYRAXMK6aMZb1ZTt59KN1fpcjUcjTUADuAH4ONLbT5iQzW2JmT5rZUI/rEYl5h47O5huj+nHXm59RvrPO73IkyngWCmY2Cyh2zuW30+w5INc5dwDwOvBwG+91oZktMLMFJSUlHlQrEluunDGW8p11/PltrdAme8bLPYXpwPFmVgA8DhxpZo82b+CcK3XO7bo2/35gSmtv5Jy7zzmX55zLy87WlEsiHdlvUCbfOXAID71fQMGWSr/LkSjiWSg4565yzg1xzuUCpwNvOufOat4mtJrbLsfT/oC0iOyBXxw7hoQ44+YX9L+VdJ7XYwq7MbMbzez40MNLzGy5mS0GLgHODXc9IrGqf0YyFx85itdXbObdVTrsKp1j0bZqU15enluwYIHfZYhEhZr6Br71x3eJCxgvX3ooCXFh/ztQIoSZ5Tvn8jpqp58QkRiWFB/HdbPGs6akUmsuSKcoFERi3JFj+3PY6GzufP0ztmjNBemAQkEkxpkZ180az866Bm5/5VO/y5EIp1AQ6QFG9k/j3ENyeWLBepYUbfO7HIlgCgWRHuKn3xxF39QkbpizXLOoSpsUCiI9RHpyAlfOGMvCddt46uMiv8uRCKVQEOlBvnPgYA4c1pvbXl7J9mrNiyS7UyiI9CCBgHHj8RMorazlztc/87sciUAKBZEeZv8hmZw+dRh//6CAVZsr/C5HIoxCQaQH+tm3xpCWFM8v5ywn2mY16KlO++uHPPJhgeefo1AQ6YGyUhP5v2NG88GaUl5c+oXf5UgHKqrrmLu2jIqaes8/S6Eg0kOdOW0Y43IyuPmFT6gMwy8b2XuFpVUA5PZN9fyzFAoiPVR8XICbZu/HpvJq7n5Li/FEsnVlwVAYlpXi+WcpFER6sLzcLE6aPIQH3vucNSU7/C5H2rBrT2F4X4WCiHjsyhljSU6I06BzBCssraRvaiLpyQmef5ZCQaSHy05P4vKjR/PeZ1t4ZbkGnSNRYWkVw8KwlwAKBREBvnfQcMYOTOem51dQVatB50hTWFoZlkFmUCiICMFB5xtnT2DDtp3co0HniFJT38Cm7dVhGWQGhYKIhEwbkcV3Jg/mvnc/Z3WxBp0jxfqynTgHuf0UCiISZlfPHEevhDiuf3aZBp0jRGFpJQDDsnT4SETCrF9aEj8/diwfrCllzuKNfpcjNL9wTXsKIuKDM6YNY+KQTG56fgXlOzW9tt8KSytJS4onKzUxLJ+nUBCRr4gLGL8+cX/KKmv4w6ta09lvhWVVDO+bgpmF5fMUCiKymwmDMzn74Fwe+aiQpUXlfpfToxWWVoXlSuZdFAoi0qrLjxlN37QkrnlmKQ1a09kXDY2Ooq1VYRtkBoWCiLQhIzmB62aNZ0lROf/4sMDvcnqkjdt2UtfgwjbIDAoFEWnHtw/I4fAx2fzulU/ZsG2n3+X0OLvOPArXFBegUBCRdpgZN82egHNw/TO6diHcCsuC1yiEa4oLUCiISAeGZqXwf8eM5o2Vxby0TBPmhVNhaRWJ8QEGZiSH7TMVCiLSoXMPyWXC4AxumLNc1y6EUWFpJcOyUggEwnM6KigURKQT4uMC/OY7B1C6o4bbXl7pdzk9RmFpFcPDNBHeLgoFEemUCYMzueDrI3hs7jrmF5T5XU7Mc86xrix86yjsolAQkU677OjRDO7di6v+u5Sa+ga/y4lpJTtqqKptCOsgM4QhFMwszswWmtnzrTyXZGZPmNlqM5trZrle1yMiey8lMZ6bT5zA6uId3Pv2536XE9P8OB0VwrOn8FNgRRvPXQBsdc6NBP4I3BaGekSkC44Y059vTxzEPW+t1roLHvpydtQY2lMwsyHAccADbTSZDTwcuv8kcJSFa9YnEdlr188aT3JCgKufXkqjpsDwxLrSSgIGg3v3Cuvner2ncAfwc6CxjecHA+sBnHP1QDnQt2UjM7vQzBaY2YKSkhKvahWRTspOT+LqmeOYt7aM/+Sv97ucmFRQWsXgPr1IjA/v0K9nn2Zms4Bi51x+e81a2bbbnx3Oufucc3nOubzs7Oxuq1FE9t6peUOZNiKLX7+wgpKKGr/LiTmFZVUMD+NEeLt4GUHTgePNrAB4HDjSzB5t0aYIGApgZvFAJqBz3USiQCBg3HLi/lTXNWr5Tg8UllaGfZAZPAwF59xVzrkhzrlc4HTgTefcWS2azQHOCd0/OdRGP1kiUWJk/zQuPXoULy37gueWbPK7nJhRvrOObVV1YZ0ddZewX6dgZjea2fGhhw8Cfc1sNXA5cGW46xGRrrnwG/swaWhvrn92GcUV1X6XExMKS4MT4YVzHYVdwhIKzrm3nXOzQvevd87NCd2vds6d4pwb6Zyb5pzTic8iUSY+LsDvT53IztoGrv7vUh1G6gZLQqvdjctJD/tn64pmEemyfbPT+Nm3xvD6imL++/EGv8uJevMLyuifnsSwMM97BAoFEekm500fwdTcPvzyueVsKteCPF0xf20ZU0dk4cdlWwoFEekWcQHj9lMmUt/g+MVTOoy0t4q2VrGxvJppuVm+fL5CQUS6zfC+qVw9cyzvrirhn3PX+V1OVNo1A+1UhYKIxIKzDhrON0b149cvrKBgS6Xf5USdeWu3kp4cz5iB4R9kBoWCiHQzM+O3Jx9AQpxx+b8XUd/Q1iw30pr5BWXkDe9DXBhXW2tOoSAi3S4nsxc3nTCBj9dt46/v6kzzziqrrGV18Q6mjvDn0BEoFETEI8dPHMRxB+Rwx+urWL6x3O9yosKu8QS/BplBoSAiHjEzbp49gd4piVz+xGKqauv9LinizV9bRmJ8gP2HZPpWg0JBRDzTJzWR20+ZyKriCi57YpHWXujA/IIyJg3tTVJ8nG81KBRExFOHjc7m2uPG88ryzdz2ykq/y4lYlTX1LNu43ddDRwDxvn66iPQI50/PZe2WHfz1nc8Z0TeV06cN87ukiLNw3TYaGp2vg8ygUBCRMDAzfvnt/SgsreLaZ5YxNCuF6SP7+V1WRJlXUEbAYPKw3r7WocNHIhIW8XEB7vnuZPbJTuWiR/P5bHOF3yVFlPlryxg/KIP05ARf61AoiEjYZCQn8OA5U0lOiON7D86jaGuV3yVFhNr6Rhau3+rb1BbNKRREJKyGZqXwj/OnUVlbz9kPzmPLDq3vvGxjOdV1jb4PMoNCQUR8MC4ng7+dO5WN5Ts596F5VFTX+V2Sr975tASAPIWCiPRUU3Oz+Mt3p7ByUwXf/8cCqusa/C7JF/mFZfz57dUcNbY/2elJfpejUBAR/xwxtj+3nzKRjz4v49LHF9HQwy5u+6K8mose/ZjBvXvxh1Mn+V0OoFAQEZ+dcOBgrps1npeXf8EtL67wu5ywqa5r4AeP5lNVU899Z+eRmeLvWUe76DoFEfHdBV8fwfqyKh58fy1D+/Ti3Okj/C7JU845rn1mGYvXb+Pes6YweoA/aye0RqEgIhHhulnj2bhtJ796/hMG9e7FMfsN9Lskz/z9gwKezC/ikqNGceyEyOqnDh+JSESICxh3nn4gBwzpzSWPL2Tx+m1+l+SJhz8o4FfPfcLR4wdw6VGj/C5nNwoFEYkYvRLjePCcPLLTkzj3oXnkF271u6Rudc9bq7lhznKOHj+Au844kIBPq6u1R6EgIhGlX1oSj5z/NTJ6JXDm/R/x0tJNfpfUZc45bnt5Jb975VNOmDSIP393MskJ/k2P3R6FgohEnNx+qfz3h4ew36AMfvTYxzzw3uc4F52nq1bV1nPds8v4y9trOPNrw/jDqZNIiIvcX70aaBaRiNQ3LYnHvn8Qlz2xiJtfWMH6siqumzWe+Aj+hdrc9uo6HvmwkAffX0tZZS0/OHQfrpwxFrPIO2TUnEJBRCJWckIc95w5mVtfWsH9761lbWkVd51xIJm9IuOc/tZs2LaTf81dx8MfFlBRXc/hY7K5+IiRETGFRWcoFEQkogUCxjXHjWef7DSuf3YZJ97zP+4/J499s9P8Lq1JcUU1Ly7ZxHNLNpFfuBUzOHa/gfz4iJFMGOzfest7w6LtOF1eXp5bsGCB32WIiA/mrS3jokfzqWto5O4zJ3PY6Gxf61laVM6db3zGmys30+hg7MB0vj1xELMOyGF431Rfa2vJzPKdc3kdtlMoiEg0Kdpaxf97eAGrNldw2TdH88PD9w37OEMwDFbx+opiMnsl8L2DhjN70iBGRdCVyS11NhR0+EhEosqQPik89cNDuOq/S/n9a6t469Ni/nDqJHL7efuXeV1DI2+tLOaxeet4+9MSMnslcMUxoznnkFzfV0vrTp7tKZhZMvAukEQwfJ50zt3Qos25wO+ADaFNdzvnHmjvfbWnICK7PLtoA9c9s4z6Rse1x43njGlDu/3snjUlO/j3gvU8lb+BLTtqyE5P4uyDhnPu9OgKg0jYU6gBjnTO7TCzBOB9M3vJOfdRi3ZPOOcu9rAOEYlRsycNZtqILK74z2KufnopcxZv4AeH7stho7P3+mrhhkbHwnVbeX1FMW+u3MyqzTuICxhHjOnPaVOHcsSY7Kg5LXZveBYKLrgLsiP0MCF0i64BDBGJeDmZvXjk/K/x6NxC7n5zNef9fT77ZKdy3vQRnDR5MCmJbf+aq29opLCsipX+TTgUAAAJJElEQVSbKlixaTsrv9hOfuFWtlbVER8wpo3I4tS8oRw/cRD9M5LD2Cv/eDrQbGZxQD4wErjHOfeLFs+fC9wKlACrgMucc+tbeZ8LgQsBhg0bNqWwsNCzmkUketXWN/LSsk08+P5alhSVEx8weiXGkRQfR1J8gKT4ADX1jVTW1lNV20BtfWPTa+MCxr7ZqUwYnMmRY/tz6OhsMqLo8FBHIursIzPrDTwN/MQ5t6zZ9r7ADudcjZldBJzqnDuyvffSmIKIdMQ5x8frtvLGimKqahuoqW+kpj74b3J8HCmJcaQkxZGSEM+g3smMy8lgZP+0iJ2PqDtEwphCE+fcNjN7GzgWWNZse2mzZvcDt4WjHhGJbWbGlOFZTBkeHVcRRxLPRkvMLDu0h4CZ9QK+Caxs0San2cPjgZ6zFp+ISATyck8hB3g4NK4QAP7tnHvezG4EFjjn5gCXmNnxQD1QBpzrYT0iItIBXdEsItIDdHZMIXZPthURkT2mUBARkSYKBRERaaJQEBGRJgoFERFpEnVnH5lZCdBynotMoLyDbe09bu1+P2BLF0ptraY9adfZ7W31o/nj5tvD0a/22sTid9XWc3vTr2j7rlpu8/q7aquGPWkTiz+Dndk+3DnX8apEzrmovwH3dbStvcet3Sd4LUW31rQn7Tq7va1+tOhL8zae96u9NrH4XXVnv6Ltu+rM99Od31W4+hVtP4N7ur29W6wcPnquE9vae9zW/a7o7Pu01a6z29ur/bk2tndFZ96rvTax+F219dze9CvavquW27z+rjr7Xj3tZ3BPt7cp6g4fhYuZLXCduNAj2sRiv2KxTxCb/YrFPkFs9StW9hS8cJ/fBXgkFvsVi32C2OxXLPYJYqhf2lMQEZEm2lMQEZEmPSIUzOxvZlZsZss6br3ba6eY2VIzW21mf7Jmq4Kb2U/M7FMzW25mv+3eqjusq9v7ZGa/NLMNZrYodJvZ/ZV3WJsn31Xo+SvMzJlZv+6ruNO1efF93WRmS0Lf1atmNqj7K2+3Li/69DszWxnq19O7pt8PJ4/6dUro90SjmUX22ENXTw+LhhtwKDAZWLYXr50HHAwY8BIwI7T9COB1ICn0uH8M9OmXwBWx9l2FnhsKvELwGpd+sdAvIKNZm0uAe2OgT8cA8aH7twG3xch3NQ4YA7wN5IW7T3ty6xF7Cs65dwmu19DEzPY1s5fNLN/M3jOzsS1fF1oEKMM596ELfrP/AE4IPf1D4DfOuZrQZxR724uv8qhPvvOwX38Efg74MojmRb+cc9ubNU0lzH3zqE+vOufqQ00/AoZ424vdedSvFc65T8NRf1f1iFBow30E14yeAlwB/LmVNoOBomaPi0LbAEYD3zCzuWb2jplN9bTazulqnwAuDu26/83M+nhX6h7pUr8suJDTBufcYq8L3UNd/r7M7Ndmth74LnC9h7V2Vnf8DO5yPsG/tiNBd/YrooVljeZIY2ZpwCHAf5oddk5qrWkr23b9NRYP9AEOAqYC/zazfUJ/IYRdN/XpL8BNocc3Ab8n+D+mb7raLzNLAa4heFgiYnTT94Vz7hrgGjO7CrgYuKGbS+207upT6L2uIbgi4z+7s8a90Z39igY9MhQI7iFtc85Nar7RgkuH5oceziH4S7L57usQYGPofhHw31AIzDOzRoLzn5R4WXg7utwn59zmZq+7H3jey4I7qav92hcYASwO/Q89BPjYzKY5577wuPb2dMfPYHOPAS/gYyjQTX0ys3OAWcBRfv2R1UJ3f1eRze9BjXDdgFyaDRwBHwCnhO4bMLGN180nuDewa+BoZmj7RcCNofujgfWErvuI4j7lNGtzGfB4LHxXLdoU4MNAs0ff16hmbX4CPBkDfToW+ATI9uM78vpnkCgYaPa9gDB9wf8CNgF1BP/Cv4DgX48vA4tDP4TXt/HaPGAZsAa4e9cvfiAReDT03MfAkTHQp0eApcASgn/55ISrP172q0UbX0LBo+/rqdD2JQTnuBkcA31aTfAPrEWhW1jPqPKwXyeG3qsG2Ay8Eu5+dfamK5pFRKRJTz77SEREWlAoiIhIE4WCiIg0USiIiEgThYKIiDRRKEhMMLMdYf68B8xsfDe9V0NoptNlZvZcRzODmllvM/tRd3y2SEs6JVVigpntcM6ldeP7xbsvJ2bzVPPazexhYJVz7tfttM8FnnfOTQhHfdKzaE9BYpaZZZvZU2Y2P3SbHto+zcw+MLOFoX/HhLafa2b/MbPngFfN7HAze9vMngzN8f/PZvPjv71rXnwz2xGamG6xmX1kZgNC2/cNPZ5vZjd2cm/mQ76cyC/NzN4ws48tOEf/7FCb3wD7hvYufhdq+7PQ5ywxs191439G6WEUChLL7gT+6JybCpwEPBDavhI41Dl3IMGZRW9p9pqDgXOcc0eGHh8IXAqMB/YBprfyOanAR865icC7wPebff6doc/vcA6c0Fw6RxG8mhygGjjROTeZ4Podvw+F0pXAGufcJOfcz8zsGGAUMA2YBEwxs0M7+jyR1vTUCfGkZ/gmML7ZzJYZZpYOZAIPm9kogrNYJjR7zWvOueZz6c9zzhUBmNkignPivN/ic2r5cvLAfODo0P2D+XJNh8eA29uos1ez984HXgttN+CW0C/4RoJ7EANaef0xodvC0OM0giHxbhufJ9ImhYLEsgBwsHNuZ/ONZnYX8JZz7sTQ8fm3mz1d2eI9aprdb6D1/2fq3JeDc221ac9O59wkM8skGC4/Bv5EcI2EbGCKc67OzAqA5FZeb8Ctzrm/7uHniuxGh48klr1KcI0BAMxs19THmcCG0P1zPfz8jwgetgI4vaPGzrlygstqXmFmCQTrLA4FwhHA8FDTCiC92UtfAc4PzfuPmQ02s/7d1AfpYRQKEitSzKyo2e1ygr9g80KDr58QnO4c4LfArWb2PyDOw5ouBS43s3lADlDe0QuccwsJzsR5OsEFZvLMbAHBvYaVoTalwP9Cp7D+zjn3KsHDUx+a2VLgSb4aGiKdplNSRTwSWvVtp3POmdnpwBnOudkdvU7ETxpTEPHOFODu0BlD2/B5aVORztCegoiINNGYgoiINFEoiIhIE4WCiIg0USiIiEgThYKIiDRRKIiISJP/D5DkGNGBZX00AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 32:31\n",
      "epoch  train_loss  valid_loss  accuracy  map3     \n",
      "1      0.997846    9271.355469  0.744080  0.812822  (08:08)\n",
      "2      0.858220    1105.838989  0.777960  0.840417  (08:07)\n",
      "3      0.696471    2100.821289  0.796980  0.852824  (08:06)\n",
      "4      0.520798    1567.136353  0.805950  0.859583  (08:08)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(4, 5e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And then I unfreeze the whole model and resume training..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making changes to batch norm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maybe you might have a need to stop tracking runnining mean an sd in your batchnorm layers - here is how to do it. You can also update the momentum with which stats are tracked in similar fashion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "for l in flatten_model(learn.model):\n",
    "    if type(l) == nn.BatchNorm1d or type(l) == nn.BatchNorm2d:\n",
    "        l.track_running_stats = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There can be a situation in which the activations of your model change quickly enough where the running stats just cannot keep up. Take a look below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 10:28\n",
      "epoch  train_loss  valid_loss  accuracy  map3    \n",
      "1      0.520851    143.771622  0.802850  0.858103  (10:28)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit(1, 5e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The validation loss does not track the train loss (often the accuracy will also be affected).\n",
    "\n",
    "There are no hard rules to follow here, the point is that you might run into a situation such as this and there is no reason to be too concerned.\n",
    "\n",
    "In general, this should go away once you start training with a lower learning rate.\n",
    "\n",
    "There is quite a bit of information online (predominantly on pytorch forums) that you can look up if you do encounter this scenario or should the problem not go away on its own.\n",
    "\n",
    "One other think to note here - the default value of momentum in pytorch for a batchnorm layer is 0.9 and 0.99 in keras. As this has an exponential effect on how stats are accumulated, this can make quite a big difference."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds, _ = learn.get_preds(ds_type=DatasetType.Test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Downloading some of the preds as I thought I might continue working on this on another machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='preds/test_preds' target='_blank'>preds/test_preds</a><br>"
      ],
      "text/plain": [
       "/home/radek/quickdraw/preds/test_preds"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import FileLink\n",
    "FileLink('preds/test_preds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_submission(preds, data_bunch.test_dl, name, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key_id</th>\n",
       "      <th>word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9982748469652562</td>\n",
       "      <td>rollerskates roller_coaster tractor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9730298586740955</td>\n",
       "      <td>drill screwdriver megaphone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9495303600236565</td>\n",
       "      <td>matches paintbrush asparagus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9076379423232894</td>\n",
       "      <td>saxophone clarinet trombone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9278326467147683</td>\n",
       "      <td>The_Mona_Lisa face mermaid</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             key_id                                 word\n",
       "0  9982748469652562  rollerskates roller_coaster tractor\n",
       "1  9730298586740955          drill screwdriver megaphone\n",
       "2  9495303600236565         matches paintbrush asparagus\n",
       "3  9076379423232894          saxophone clarinet trombone\n",
       "4  9278326467147683           The_Mona_Lisa face mermaid"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(f'subs/{name}.csv.gz').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████| 1.50M/1.50M [00:00<00:00, 6.02MB/s]\n",
      "Successfully submitted to Quick, Draw! Doodle Recognition Challenge"
     ]
    }
   ],
   "source": [
    "!kaggle competitions submit -c quickdraw-doodle-recognition -f subs/{name}.csv.gz -m \"{name}\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}