chess-classification-hw/nbs-demo/demo-custom-cb+metric-1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add Callbacks + Metrics\n",
    "Adding more setup, by-epoch, teardown methods to learner\n",
    "\n",
    "Using new callback `TestSetRecorder` - we run validate on a third dataset. (I still need to figure out if tfms's are being applied to this third dataset...it should if test_accuracy matches my_test_metrics on an accurate model)\n",
    "\n",
    "Using `subcat_color_acc`, `subcat_piece_acc` we add metrics to be monitored at each epoch, for correctly predicting a piece's player-color and piece-class individually. What could this show?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai2.vision.all import *\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from modules.trainutils import (build_dl, \n",
    "                                piece_class_parse,\n",
    "                                my_metrics, \n",
    "                                my_test_metrics,\n",
    "                                stratify_sample)\n",
    "\n",
    "from modules.trainutils import (subcat_color_acc,\n",
    "                                subcat_piece_acc)\n",
    "\n",
    "# from modules.learnutils import TestSetRecorder\n",
    "\n",
    "test_path = Path('../../../other-chess-data/regulation-test-all/')\n",
    "train_path = Path('../../../rf-chess-data/cropped_v1/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "228"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dl = build_dl(test_path, n=None)\n",
    "len(test_dl.train.items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 200\n",
    "seed=42\n",
    "\n",
    "train_dl = ImageDataLoaders.from_name_func(\n",
    "                train_path, \n",
    "                # get_image_files(train_path),\n",
    "                stratify_sample(train_path, n=n, np_seed=seed),\n",
    "                valid_pct=0.2, \n",
    "                seed=42,\n",
    "                label_func=piece_class_parse, \n",
    "                item_tfms=RandomResizedCrop(128, min_scale=0.5),\n",
    "                batch_tfms=aug_transforms(),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Setup Custom Learner\n",
    " - add new metrics\n",
    " - add third 'dl' to learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = cnn_learner(train_dl, resnet18, \n",
    "                    metrics=[accuracy, \n",
    "                             subcat_color_acc, \n",
    "                             subcat_piece_acc,\n",
    "                            ]\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_dl = DataLoaders(learn.dls[0], learn.dls[1], test_dl.train)\n",
    "learn.dls = new_dl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(154, 38, 228)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(learn.dls[0].items), len(learn.dls[1].items), len(learn.dls[2].items)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Build callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestSetRecorder(Callback):\n",
    "    def __init__(self, ds_idx=2, **kwargs):\n",
    "        self.values = []\n",
    "        self.ds_idx = ds_idx\n",
    "    \n",
    "    def after_epoch(self):\n",
    "        old_log = self.recorder.log.copy()\n",
    "        self.learn._do_epoch_validate(ds_idx=self.ds_idx, dl=None)\n",
    "        self.values.append(self.recorder.log[len(old_log):])\n",
    "        self.recorder.log = old_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai2.learner.Learner at 0x7f7c68005bd0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# learn.recorder.train_metrics = True\n",
    "learn.add_cb(TestSetRecorder(ds_idsx=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>subcat_color_acc</th>\n",
       "      <th>subcat_piece_acc</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.952985</td>\n",
       "      <td>9.220811</td>\n",
       "      <td>0.078947</td>\n",
       "      <td>0.447368</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.596785</td>\n",
       "      <td>4.932112</td>\n",
       "      <td>0.078947</td>\n",
       "      <td>0.394737</td>\n",
       "      <td>0.078947</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.243423</td>\n",
       "      <td>2.496461</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.578947</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.932016</td>\n",
       "      <td>1.539852</td>\n",
       "      <td>0.342105</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.621784</td>\n",
       "      <td>1.154464</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>0.921053</td>\n",
       "      <td>0.578947</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2.421329</td>\n",
       "      <td>1.041189</td>\n",
       "      <td>0.578947</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.605263</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Examine + Format values within callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cb_index(learn, cb_name):\n",
    "    return [i for i, e in enumerate(list(learn.cbs)) \n",
    "            if e.__class__.__name__ == cb_name][0]\n",
    "\n",
    "def get_cb(learn, cb_name):\n",
    "    return learn.cbs[get_cb_index(learn, cb_name)]\n",
    "\n",
    "first_recorder = get_cb(learn, 'Recorder')\n",
    "second_recorder = get_cb(learn, 'TestSetRecorder')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#4) [TrainEvalCallback,Recorder,ProgressCallback,TestSetRecorder]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.cbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_recorder = pd.DataFrame(first_recorder.values, \n",
    "                       columns = first_recorder.metric_names[1:-1])\n",
    "\n",
    "met_names = [e.name for e in first_recorder.metrics]\n",
    "cols = ['loss']\n",
    "cols += met_names\n",
    "cols = ['test_' + col for col in cols]\n",
    "\n",
    "df_second_recorder = pd.DataFrame(second_recorder.values, \n",
    "                                  columns = cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>subcat_color_acc</th>\n",
       "      <th>subcat_piece_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.952985</td>\n",
       "      <td>9.220811</td>\n",
       "      <td>0.078947</td>\n",
       "      <td>0.447368</td>\n",
       "      <td>0.105263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.596785</td>\n",
       "      <td>4.932112</td>\n",
       "      <td>0.078947</td>\n",
       "      <td>0.394737</td>\n",
       "      <td>0.078947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.243423</td>\n",
       "      <td>2.496461</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.578947</td>\n",
       "      <td>0.421053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.932016</td>\n",
       "      <td>1.539852</td>\n",
       "      <td>0.342105</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>0.421053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.621784</td>\n",
       "      <td>1.154464</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>0.921053</td>\n",
       "      <td>0.578947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.421329</td>\n",
       "      <td>1.041189</td>\n",
       "      <td>0.578947</td>\n",
       "      <td>0.973684</td>\n",
       "      <td>0.605263</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_loss  valid_loss  accuracy  subcat_color_acc  subcat_piece_acc\n",
       "0    3.952985    9.220811  0.078947          0.447368          0.105263\n",
       "1    3.596785    4.932112  0.078947          0.394737          0.078947\n",
       "2    3.243423    2.496461  0.263158          0.578947          0.421053\n",
       "3    2.932016    1.539852  0.342105          0.789474          0.421053\n",
       "4    2.621784    1.154464  0.552632          0.921053          0.578947\n",
       "5    2.421329    1.041189  0.578947          0.973684          0.605263"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_recorder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_subcat_color_acc</th>\n",
       "      <th>test_subcat_piece_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.132629</td>\n",
       "      <td>0.100877</td>\n",
       "      <td>0.464912</td>\n",
       "      <td>0.223684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.947631</td>\n",
       "      <td>0.109649</td>\n",
       "      <td>0.464912</td>\n",
       "      <td>0.232456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.667915</td>\n",
       "      <td>0.144737</td>\n",
       "      <td>0.521930</td>\n",
       "      <td>0.267544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.776058</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.557018</td>\n",
       "      <td>0.223684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.969409</td>\n",
       "      <td>0.114035</td>\n",
       "      <td>0.543860</td>\n",
       "      <td>0.223684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3.178069</td>\n",
       "      <td>0.131579</td>\n",
       "      <td>0.530702</td>\n",
       "      <td>0.228070</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   test_loss  test_accuracy  test_subcat_color_acc  test_subcat_piece_acc\n",
       "0   4.132629       0.100877               0.464912               0.223684\n",
       "1   2.947631       0.109649               0.464912               0.232456\n",
       "2   2.667915       0.144737               0.521930               0.267544\n",
       "3   2.776058       0.105263               0.557018               0.223684\n",
       "4   2.969409       0.114035               0.543860               0.223684\n",
       "5   3.178069       0.131579               0.530702               0.228070"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_second_recorder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Verify this works\n",
    "Does the accuracy calc'd in the SecondRecorder cb match another method?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1315789520740509"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_acc_final = df_second_recorder.loc[5,'test_accuracy']\n",
    "test_acc_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.09649122807017543"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_metrics = my_test_metrics(learn, test_path)\n",
    "test_acc_final_2 = test_metrics[1]\n",
    "test_acc_final_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OK, so they are slightly different, but still it's clearly from the test set, because accuracy from train/valid set is 50% by final epoch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:fastai2]",
   "language": "python",
   "name": "conda-env-fastai2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}