common-feature-extractors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.IMAGENETTE_160)\n",
    "data_nette = (ImageList.from_folder(path).split_by_folder(valid='val')\n",
    "              .label_from_folder()\n",
    "              .transform(get_transforms(), size=128)\n",
    "              .databunch(bs=64)\n",
    "              .normalize(imagenet_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.441065</td>\n",
       "      <td>0.201856</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.244970</td>\n",
       "      <td>0.138996</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.179425</td>\n",
       "      <td>0.131967</td>\n",
       "      <td>00:10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = cnn_learner(data_nette, models.mobilenet_v2, concat_pool=False, pretrained=True, train_bn=True)\n",
    "learn.fit_one_cycle(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.140665</td>\n",
       "      <td>0.131163</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.180994</td>\n",
       "      <td>0.129539</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.131018</td>\n",
       "      <td>0.118751</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.100560</td>\n",
       "      <td>0.112933</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.081557</td>\n",
       "      <td>0.112444</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(5, slice(1e-6,1e-3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "feature_extractor = copy.deepcopy(learn.model[0])\n",
    "feature_extractor.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 1280, 7, 7])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.rand(2,3,224,224).cuda()\n",
    "feat_out = feature_extractor(x)\n",
    "feat_out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.PETS)\n",
    "path_img = path/'images'\n",
    "fnames = get_image_files(path_img)\n",
    "pat = r'/([^/]+)_\\d+.jpg$'\n",
    "\n",
    "\n",
    "data_pets = ImageDataBunch.from_name_re(path_img, fnames, pat, ds_tfms=get_transforms(), size=128, bs=64\n",
    "                                        ).normalize(imagenet_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.379401</td>\n",
       "      <td>0.716572</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pets_learner = cnn_learner(data_pets, models.mobilenet_v2, train_bn=False)\n",
    "pets_learner.model[0] = feature_extractor # this changes the `feature extractor`, as shown below\n",
    "pets_learner.fit(1)\n",
    "pets_learner.model[0].eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `pets_learner.model[0]` is now sharing memory with `feature_extractor`, and changes it as it trains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.equal(feat_out, pets_learner.model[0](x))\n",
    "torch.equal(feat_out, feature_extractor(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def models_equal(model_1, model_2):\n",
    "    models_differ = 0\n",
    "    for key_item_1, key_item_2 in zip(model_1.state_dict().items(), model_2.state_dict().items()):\n",
    "        if torch.equal(key_item_1[1], key_item_2[1]):\n",
    "            pass\n",
    "        else:\n",
    "            models_differ += 1\n",
    "            if (key_item_1[0] == key_item_2[0]):\n",
    "                print('Mismtach found at', key_item_1[0])\n",
    "            else:\n",
    "                raise Exception\n",
    "                print('Models being compared have different architectures')\n",
    "    if models_differ == 0:\n",
    "        print('Models match perfectly!')\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mismtach found at 0.0.1.running_mean\n",
      "Mismtach found at 0.0.1.running_var\n",
      "Mismtach found at 0.0.1.num_batches_tracked\n",
      "Mismtach found at 0.1.conv.0.1.running_mean\n",
      "Mismtach found at 0.1.conv.0.1.running_var\n",
      "Mismtach found at 0.1.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.1.conv.2.running_mean\n",
      "Mismtach found at 0.1.conv.2.running_var\n",
      "Mismtach found at 0.1.conv.2.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.0.1.running_mean\n",
      "Mismtach found at 0.2.conv.0.1.running_var\n",
      "Mismtach found at 0.2.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.1.1.running_mean\n",
      "Mismtach found at 0.2.conv.1.1.running_var\n",
      "Mismtach found at 0.2.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.3.running_mean\n",
      "Mismtach found at 0.2.conv.3.running_var\n",
      "Mismtach found at 0.2.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.0.1.running_mean\n",
      "Mismtach found at 0.3.conv.0.1.running_var\n",
      "Mismtach found at 0.3.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.1.1.running_mean\n",
      "Mismtach found at 0.3.conv.1.1.running_var\n",
      "Mismtach found at 0.3.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.3.running_mean\n",
      "Mismtach found at 0.3.conv.3.running_var\n",
      "Mismtach found at 0.3.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.0.1.running_mean\n",
      "Mismtach found at 0.4.conv.0.1.running_var\n",
      "Mismtach found at 0.4.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.1.1.running_mean\n",
      "Mismtach found at 0.4.conv.1.1.running_var\n",
      "Mismtach found at 0.4.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.3.running_mean\n",
      "Mismtach found at 0.4.conv.3.running_var\n",
      "Mismtach found at 0.4.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.0.1.running_mean\n",
      "Mismtach found at 0.5.conv.0.1.running_var\n",
      "Mismtach found at 0.5.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.1.1.running_mean\n",
      "Mismtach found at 0.5.conv.1.1.running_var\n",
      "Mismtach found at 0.5.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.3.running_mean\n",
      "Mismtach found at 0.5.conv.3.running_var\n",
      "Mismtach found at 0.5.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.0.1.running_mean\n",
      "Mismtach found at 0.6.conv.0.1.running_var\n",
      "Mismtach found at 0.6.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.1.1.running_mean\n",
      "Mismtach found at 0.6.conv.1.1.running_var\n",
      "Mismtach found at 0.6.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.3.running_mean\n",
      "Mismtach found at 0.6.conv.3.running_var\n",
      "Mismtach found at 0.6.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.0.1.running_mean\n",
      "Mismtach found at 0.7.conv.0.1.running_var\n",
      "Mismtach found at 0.7.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.1.1.running_mean\n",
      "Mismtach found at 0.7.conv.1.1.running_var\n",
      "Mismtach found at 0.7.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.3.running_mean\n",
      "Mismtach found at 0.7.conv.3.running_var\n",
      "Mismtach found at 0.7.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.0.1.running_mean\n",
      "Mismtach found at 0.8.conv.0.1.running_var\n",
      "Mismtach found at 0.8.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.1.1.running_mean\n",
      "Mismtach found at 0.8.conv.1.1.running_var\n",
      "Mismtach found at 0.8.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.3.running_mean\n",
      "Mismtach found at 0.8.conv.3.running_var\n",
      "Mismtach found at 0.8.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.0.1.running_mean\n",
      "Mismtach found at 0.9.conv.0.1.running_var\n",
      "Mismtach found at 0.9.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.1.1.running_mean\n",
      "Mismtach found at 0.9.conv.1.1.running_var\n",
      "Mismtach found at 0.9.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.3.running_mean\n",
      "Mismtach found at 0.9.conv.3.running_var\n",
      "Mismtach found at 0.9.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.0.1.running_mean\n",
      "Mismtach found at 0.10.conv.0.1.running_var\n",
      "Mismtach found at 0.10.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.1.1.running_mean\n",
      "Mismtach found at 0.10.conv.1.1.running_var\n",
      "Mismtach found at 0.10.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.3.running_mean\n",
      "Mismtach found at 0.10.conv.3.running_var\n",
      "Mismtach found at 0.10.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.0.1.running_mean\n",
      "Mismtach found at 0.11.conv.0.1.running_var\n",
      "Mismtach found at 0.11.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.1.1.running_mean\n",
      "Mismtach found at 0.11.conv.1.1.running_var\n",
      "Mismtach found at 0.11.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.3.running_mean\n",
      "Mismtach found at 0.11.conv.3.running_var\n",
      "Mismtach found at 0.11.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.0.1.running_mean\n",
      "Mismtach found at 0.12.conv.0.1.running_var\n",
      "Mismtach found at 0.12.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.1.1.running_mean\n",
      "Mismtach found at 0.12.conv.1.1.running_var\n",
      "Mismtach found at 0.12.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.3.running_mean\n",
      "Mismtach found at 0.12.conv.3.running_var\n",
      "Mismtach found at 0.12.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.0.1.running_mean\n",
      "Mismtach found at 0.13.conv.0.1.running_var\n",
      "Mismtach found at 0.13.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.1.1.running_mean\n",
      "Mismtach found at 0.13.conv.1.1.running_var\n",
      "Mismtach found at 0.13.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.3.running_mean\n",
      "Mismtach found at 0.13.conv.3.running_var\n",
      "Mismtach found at 0.13.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.0.1.running_mean\n",
      "Mismtach found at 0.14.conv.0.1.running_var\n",
      "Mismtach found at 0.14.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.1.1.running_mean\n",
      "Mismtach found at 0.14.conv.1.1.running_var\n",
      "Mismtach found at 0.14.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.3.running_mean\n",
      "Mismtach found at 0.14.conv.3.running_var\n",
      "Mismtach found at 0.14.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.0.1.running_mean\n",
      "Mismtach found at 0.15.conv.0.1.running_var\n",
      "Mismtach found at 0.15.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.1.1.running_mean\n",
      "Mismtach found at 0.15.conv.1.1.running_var\n",
      "Mismtach found at 0.15.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.3.running_mean\n",
      "Mismtach found at 0.15.conv.3.running_var\n",
      "Mismtach found at 0.15.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.0.1.running_mean\n",
      "Mismtach found at 0.16.conv.0.1.running_var\n",
      "Mismtach found at 0.16.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.1.1.running_mean\n",
      "Mismtach found at 0.16.conv.1.1.running_var\n",
      "Mismtach found at 0.16.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.3.running_mean\n",
      "Mismtach found at 0.16.conv.3.running_var\n",
      "Mismtach found at 0.16.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.0.1.running_mean\n",
      "Mismtach found at 0.17.conv.0.1.running_var\n",
      "Mismtach found at 0.17.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.1.1.running_mean\n",
      "Mismtach found at 0.17.conv.1.1.running_var\n",
      "Mismtach found at 0.17.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.3.running_mean\n",
      "Mismtach found at 0.17.conv.3.running_var\n",
      "Mismtach found at 0.17.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.18.1.running_mean\n",
      "Mismtach found at 0.18.1.running_var\n",
      "Mismtach found at 0.18.1.num_batches_tracked\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models_equal(feature_extractor, learn.model[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mismtach found at 0.0.1.running_mean\n",
      "Mismtach found at 0.0.1.running_var\n",
      "Mismtach found at 0.0.1.num_batches_tracked\n",
      "Mismtach found at 0.1.conv.0.1.running_mean\n",
      "Mismtach found at 0.1.conv.0.1.running_var\n",
      "Mismtach found at 0.1.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.1.conv.2.running_mean\n",
      "Mismtach found at 0.1.conv.2.running_var\n",
      "Mismtach found at 0.1.conv.2.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.0.1.running_mean\n",
      "Mismtach found at 0.2.conv.0.1.running_var\n",
      "Mismtach found at 0.2.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.1.1.running_mean\n",
      "Mismtach found at 0.2.conv.1.1.running_var\n",
      "Mismtach found at 0.2.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.3.running_mean\n",
      "Mismtach found at 0.2.conv.3.running_var\n",
      "Mismtach found at 0.2.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.0.1.running_mean\n",
      "Mismtach found at 0.3.conv.0.1.running_var\n",
      "Mismtach found at 0.3.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.1.1.running_mean\n",
      "Mismtach found at 0.3.conv.1.1.running_var\n",
      "Mismtach found at 0.3.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.3.running_mean\n",
      "Mismtach found at 0.3.conv.3.running_var\n",
      "Mismtach found at 0.3.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.0.1.running_mean\n",
      "Mismtach found at 0.4.conv.0.1.running_var\n",
      "Mismtach found at 0.4.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.1.1.running_mean\n",
      "Mismtach found at 0.4.conv.1.1.running_var\n",
      "Mismtach found at 0.4.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.3.running_mean\n",
      "Mismtach found at 0.4.conv.3.running_var\n",
      "Mismtach found at 0.4.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.0.1.running_mean\n",
      "Mismtach found at 0.5.conv.0.1.running_var\n",
      "Mismtach found at 0.5.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.1.1.running_mean\n",
      "Mismtach found at 0.5.conv.1.1.running_var\n",
      "Mismtach found at 0.5.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.3.running_mean\n",
      "Mismtach found at 0.5.conv.3.running_var\n",
      "Mismtach found at 0.5.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.0.1.running_mean\n",
      "Mismtach found at 0.6.conv.0.1.running_var\n",
      "Mismtach found at 0.6.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.1.1.running_mean\n",
      "Mismtach found at 0.6.conv.1.1.running_var\n",
      "Mismtach found at 0.6.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.3.running_mean\n",
      "Mismtach found at 0.6.conv.3.running_var\n",
      "Mismtach found at 0.6.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.0.1.running_mean\n",
      "Mismtach found at 0.7.conv.0.1.running_var\n",
      "Mismtach found at 0.7.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.1.1.running_mean\n",
      "Mismtach found at 0.7.conv.1.1.running_var\n",
      "Mismtach found at 0.7.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.3.running_mean\n",
      "Mismtach found at 0.7.conv.3.running_var\n",
      "Mismtach found at 0.7.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.0.1.running_mean\n",
      "Mismtach found at 0.8.conv.0.1.running_var\n",
      "Mismtach found at 0.8.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.1.1.running_mean\n",
      "Mismtach found at 0.8.conv.1.1.running_var\n",
      "Mismtach found at 0.8.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.3.running_mean\n",
      "Mismtach found at 0.8.conv.3.running_var\n",
      "Mismtach found at 0.8.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.0.1.running_mean\n",
      "Mismtach found at 0.9.conv.0.1.running_var\n",
      "Mismtach found at 0.9.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.1.1.running_mean\n",
      "Mismtach found at 0.9.conv.1.1.running_var\n",
      "Mismtach found at 0.9.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.3.running_mean\n",
      "Mismtach found at 0.9.conv.3.running_var\n",
      "Mismtach found at 0.9.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.0.1.running_mean\n",
      "Mismtach found at 0.10.conv.0.1.running_var\n",
      "Mismtach found at 0.10.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.1.1.running_mean\n",
      "Mismtach found at 0.10.conv.1.1.running_var\n",
      "Mismtach found at 0.10.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.3.running_mean\n",
      "Mismtach found at 0.10.conv.3.running_var\n",
      "Mismtach found at 0.10.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.0.1.running_mean\n",
      "Mismtach found at 0.11.conv.0.1.running_var\n",
      "Mismtach found at 0.11.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.1.1.running_mean\n",
      "Mismtach found at 0.11.conv.1.1.running_var\n",
      "Mismtach found at 0.11.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.3.running_mean\n",
      "Mismtach found at 0.11.conv.3.running_var\n",
      "Mismtach found at 0.11.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.0.1.running_mean\n",
      "Mismtach found at 0.12.conv.0.1.running_var\n",
      "Mismtach found at 0.12.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.1.1.running_mean\n",
      "Mismtach found at 0.12.conv.1.1.running_var\n",
      "Mismtach found at 0.12.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.3.running_mean\n",
      "Mismtach found at 0.12.conv.3.running_var\n",
      "Mismtach found at 0.12.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.0.1.running_mean\n",
      "Mismtach found at 0.13.conv.0.1.running_var\n",
      "Mismtach found at 0.13.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.1.1.running_mean\n",
      "Mismtach found at 0.13.conv.1.1.running_var\n",
      "Mismtach found at 0.13.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.3.running_mean\n",
      "Mismtach found at 0.13.conv.3.running_var\n",
      "Mismtach found at 0.13.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.0.1.running_mean\n",
      "Mismtach found at 0.14.conv.0.1.running_var\n",
      "Mismtach found at 0.14.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.1.1.running_mean\n",
      "Mismtach found at 0.14.conv.1.1.running_var\n",
      "Mismtach found at 0.14.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.3.running_mean\n",
      "Mismtach found at 0.14.conv.3.running_var\n",
      "Mismtach found at 0.14.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.0.1.running_mean\n",
      "Mismtach found at 0.15.conv.0.1.running_var\n",
      "Mismtach found at 0.15.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.1.1.running_mean\n",
      "Mismtach found at 0.15.conv.1.1.running_var\n",
      "Mismtach found at 0.15.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.3.running_mean\n",
      "Mismtach found at 0.15.conv.3.running_var\n",
      "Mismtach found at 0.15.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.0.1.running_mean\n",
      "Mismtach found at 0.16.conv.0.1.running_var\n",
      "Mismtach found at 0.16.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.1.1.running_mean\n",
      "Mismtach found at 0.16.conv.1.1.running_var\n",
      "Mismtach found at 0.16.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.3.running_mean\n",
      "Mismtach found at 0.16.conv.3.running_var\n",
      "Mismtach found at 0.16.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.0.1.running_mean\n",
      "Mismtach found at 0.17.conv.0.1.running_var\n",
      "Mismtach found at 0.17.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.1.1.running_mean\n",
      "Mismtach found at 0.17.conv.1.1.running_var\n",
      "Mismtach found at 0.17.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.3.running_mean\n",
      "Mismtach found at 0.17.conv.3.running_var\n",
      "Mismtach found at 0.17.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.18.1.running_mean\n",
      "Mismtach found at 0.18.1.running_var\n",
      "Mismtach found at 0.18.1.num_batches_tracked\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models_equal(learn.model[0], pets_learner.model[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `pets_learner.model[0]` is now a copy of the `feature_extractor`, and despite being frozen and `train_bn=False`, `pets_learner.model[0]` changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.363092</td>\n",
       "      <td>0.770066</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_extractor = copy.deepcopy(learn.model[0])\n",
    "feature_extractor.eval();\n",
    "\n",
    "pets_learner = cnn_learner(data_pets, models.mobilenet_v2, train_bn=False)\n",
    "pets_learner.model[0] = copy.deepcopy(feature_extractor)\n",
    "pets_learner.fit(1)\n",
    "pets_learner.model[0].eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.equal(feat_out, pets_learner.model[0](x))\n",
    "torch.equal(feat_out, feature_extractor(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Models match perfectly!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models_equal(feature_extractor, learn.model[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mismtach found at 0.0.1.running_mean\n",
      "Mismtach found at 0.0.1.running_var\n",
      "Mismtach found at 0.0.1.num_batches_tracked\n",
      "Mismtach found at 0.1.conv.0.1.running_mean\n",
      "Mismtach found at 0.1.conv.0.1.running_var\n",
      "Mismtach found at 0.1.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.1.conv.2.running_mean\n",
      "Mismtach found at 0.1.conv.2.running_var\n",
      "Mismtach found at 0.1.conv.2.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.0.1.running_mean\n",
      "Mismtach found at 0.2.conv.0.1.running_var\n",
      "Mismtach found at 0.2.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.1.1.running_mean\n",
      "Mismtach found at 0.2.conv.1.1.running_var\n",
      "Mismtach found at 0.2.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.2.conv.3.running_mean\n",
      "Mismtach found at 0.2.conv.3.running_var\n",
      "Mismtach found at 0.2.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.0.1.running_mean\n",
      "Mismtach found at 0.3.conv.0.1.running_var\n",
      "Mismtach found at 0.3.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.1.1.running_mean\n",
      "Mismtach found at 0.3.conv.1.1.running_var\n",
      "Mismtach found at 0.3.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.3.conv.3.running_mean\n",
      "Mismtach found at 0.3.conv.3.running_var\n",
      "Mismtach found at 0.3.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.0.1.running_mean\n",
      "Mismtach found at 0.4.conv.0.1.running_var\n",
      "Mismtach found at 0.4.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.1.1.running_mean\n",
      "Mismtach found at 0.4.conv.1.1.running_var\n",
      "Mismtach found at 0.4.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.4.conv.3.running_mean\n",
      "Mismtach found at 0.4.conv.3.running_var\n",
      "Mismtach found at 0.4.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.0.1.running_mean\n",
      "Mismtach found at 0.5.conv.0.1.running_var\n",
      "Mismtach found at 0.5.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.1.1.running_mean\n",
      "Mismtach found at 0.5.conv.1.1.running_var\n",
      "Mismtach found at 0.5.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.5.conv.3.running_mean\n",
      "Mismtach found at 0.5.conv.3.running_var\n",
      "Mismtach found at 0.5.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.0.1.running_mean\n",
      "Mismtach found at 0.6.conv.0.1.running_var\n",
      "Mismtach found at 0.6.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.1.1.running_mean\n",
      "Mismtach found at 0.6.conv.1.1.running_var\n",
      "Mismtach found at 0.6.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.6.conv.3.running_mean\n",
      "Mismtach found at 0.6.conv.3.running_var\n",
      "Mismtach found at 0.6.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.0.1.running_mean\n",
      "Mismtach found at 0.7.conv.0.1.running_var\n",
      "Mismtach found at 0.7.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.1.1.running_mean\n",
      "Mismtach found at 0.7.conv.1.1.running_var\n",
      "Mismtach found at 0.7.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.7.conv.3.running_mean\n",
      "Mismtach found at 0.7.conv.3.running_var\n",
      "Mismtach found at 0.7.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.0.1.running_mean\n",
      "Mismtach found at 0.8.conv.0.1.running_var\n",
      "Mismtach found at 0.8.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.1.1.running_mean\n",
      "Mismtach found at 0.8.conv.1.1.running_var\n",
      "Mismtach found at 0.8.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.8.conv.3.running_mean\n",
      "Mismtach found at 0.8.conv.3.running_var\n",
      "Mismtach found at 0.8.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.0.1.running_mean\n",
      "Mismtach found at 0.9.conv.0.1.running_var\n",
      "Mismtach found at 0.9.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.1.1.running_mean\n",
      "Mismtach found at 0.9.conv.1.1.running_var\n",
      "Mismtach found at 0.9.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.9.conv.3.running_mean\n",
      "Mismtach found at 0.9.conv.3.running_var\n",
      "Mismtach found at 0.9.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.0.1.running_mean\n",
      "Mismtach found at 0.10.conv.0.1.running_var\n",
      "Mismtach found at 0.10.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.1.1.running_mean\n",
      "Mismtach found at 0.10.conv.1.1.running_var\n",
      "Mismtach found at 0.10.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.10.conv.3.running_mean\n",
      "Mismtach found at 0.10.conv.3.running_var\n",
      "Mismtach found at 0.10.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.0.1.running_mean\n",
      "Mismtach found at 0.11.conv.0.1.running_var\n",
      "Mismtach found at 0.11.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.1.1.running_mean\n",
      "Mismtach found at 0.11.conv.1.1.running_var\n",
      "Mismtach found at 0.11.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.11.conv.3.running_mean\n",
      "Mismtach found at 0.11.conv.3.running_var\n",
      "Mismtach found at 0.11.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.0.1.running_mean\n",
      "Mismtach found at 0.12.conv.0.1.running_var\n",
      "Mismtach found at 0.12.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.1.1.running_mean\n",
      "Mismtach found at 0.12.conv.1.1.running_var\n",
      "Mismtach found at 0.12.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.12.conv.3.running_mean\n",
      "Mismtach found at 0.12.conv.3.running_var\n",
      "Mismtach found at 0.12.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.0.1.running_mean\n",
      "Mismtach found at 0.13.conv.0.1.running_var\n",
      "Mismtach found at 0.13.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.1.1.running_mean\n",
      "Mismtach found at 0.13.conv.1.1.running_var\n",
      "Mismtach found at 0.13.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.13.conv.3.running_mean\n",
      "Mismtach found at 0.13.conv.3.running_var\n",
      "Mismtach found at 0.13.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.0.1.running_mean\n",
      "Mismtach found at 0.14.conv.0.1.running_var\n",
      "Mismtach found at 0.14.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.1.1.running_mean\n",
      "Mismtach found at 0.14.conv.1.1.running_var\n",
      "Mismtach found at 0.14.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.14.conv.3.running_mean\n",
      "Mismtach found at 0.14.conv.3.running_var\n",
      "Mismtach found at 0.14.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.0.1.running_mean\n",
      "Mismtach found at 0.15.conv.0.1.running_var\n",
      "Mismtach found at 0.15.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.1.1.running_mean\n",
      "Mismtach found at 0.15.conv.1.1.running_var\n",
      "Mismtach found at 0.15.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.15.conv.3.running_mean\n",
      "Mismtach found at 0.15.conv.3.running_var\n",
      "Mismtach found at 0.15.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.0.1.running_mean\n",
      "Mismtach found at 0.16.conv.0.1.running_var\n",
      "Mismtach found at 0.16.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.1.1.running_mean\n",
      "Mismtach found at 0.16.conv.1.1.running_var\n",
      "Mismtach found at 0.16.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.16.conv.3.running_mean\n",
      "Mismtach found at 0.16.conv.3.running_var\n",
      "Mismtach found at 0.16.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.0.1.running_mean\n",
      "Mismtach found at 0.17.conv.0.1.running_var\n",
      "Mismtach found at 0.17.conv.0.1.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.1.1.running_mean\n",
      "Mismtach found at 0.17.conv.1.1.running_var\n",
      "Mismtach found at 0.17.conv.1.1.num_batches_tracked\n",
      "Mismtach found at 0.17.conv.3.running_mean\n",
      "Mismtach found at 0.17.conv.3.running_var\n",
      "Mismtach found at 0.17.conv.3.num_batches_tracked\n",
      "Mismtach found at 0.18.1.running_mean\n",
      "Mismtach found at 0.18.1.running_var\n",
      "Mismtach found at 0.18.1.num_batches_tracked\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models_equal(learn.model[0], pets_learner.model[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How To Make Models Share The Same Feature Extractor Reliably?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}