vision/imagenette/imagewoof2x2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fastai2\n",
    "from fastai2.basics import *\n",
    "from fastai2.callback.all import *\n",
    "from fastai2.vision.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbl2name = dict(\n",
    "  n02086240= 'Shih-Tzu',\n",
    "  n02087394= 'Rhodesian ridgeback',\n",
    "  n02088364= 'Beagle',\n",
    "  n02089973= 'English foxhound',\n",
    "  n02093754= 'Australian terrier',\n",
    "  n02096294= 'Border terrier',\n",
    "  n02099601= 'Golden retriever',\n",
    "  n02105641= 'Old English sheepdog',\n",
    "  n02111889= 'Samoyed',\n",
    "  n02115641= 'Dingo'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#2) [/home/lgvaz/.fastai/data/imagewoof/val,/home/lgvaz/.fastai/data/imagewoof/train]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = untar_data(URLs.IMAGEWOOF); path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dblock = DataBlock(blocks=(ImageBlock, CategoryBlock),\n",
    "          get_items=get_image_files,\n",
    "          splitter=GrandparentSplitter(valid_name='val'),\n",
    "          get_y=[parent_label, lbl2name.get])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbunch = dblock.databunch(path,\n",
    "                          bs=64,\n",
    "                          item_tfms=[FlipItem(), RandomResizedCrop(128, 0.35)],\n",
    "                          batch_tfms=[Normalize(*imagenet_stats)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _conv_func(ndim=2, transpose=False):\n",
    "    \"Return the proper conv `ndim` function, potentially `transposed`.\"\n",
    "    assert 1 <= ndim <=3\n",
    "    return getattr(nn, f'Conv{\"Transpose\" if transpose else \"\"}{ndim}d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvLayer(nn.Sequential):\n",
    "    \"Create a sequence of convolutional (`ni` to `nf`), ReLU (if `use_activ`) and `norm_type` layers.\"\n",
    "    def __init__(self, ni, nf, ks=3, stride=1, padding=None, bias=None, ndim=2, norm_type=NormType.Batch, bn_1st=True,\n",
    "                 act_cls=defaults.activation, transpose=False, init=nn.init.kaiming_normal_, xtra=None, **kwargs):\n",
    "#         if padding is None: padding = ((ks-1)//2 if not transpose else 0)\n",
    "        layers = []\n",
    "        if padding is None:\n",
    "            if   ks==1: pass\n",
    "            elif ks==2: layers.append(nn.ZeroPad2d((1, 0, 1, 0)))\n",
    "            else:       raise ValueError(f'ks={ks}')\n",
    "        bn = norm_type in (NormType.Batch, NormType.BatchZero)\n",
    "        if bias is None: bias = not bn\n",
    "        conv_func = _conv_func(ndim, transpose=transpose)\n",
    "        conv = init_default(conv_func(ni, nf, kernel_size=ks, bias=bias, stride=stride, **kwargs), init)\n",
    "        if   norm_type==NormType.Weight:   conv = weight_norm(conv)\n",
    "        elif norm_type==NormType.Spectral: conv = spectral_norm(conv)\n",
    "        layers += [conv]\n",
    "        act_bn = []\n",
    "        if act_cls is not None: act_bn.append(act_cls())\n",
    "        if bn: act_bn.append(BatchNorm(nf, norm_type=norm_type, ndim=ndim))\n",
    "        if bn_1st: act_bn.reverse()\n",
    "        layers += act_bn\n",
    "        if xtra: layers.append(xtra)\n",
    "        super().__init__(*layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResBlock(nn.Module):\n",
    "    \"Resnet block from `ni` to `nh` with `stride`\"\n",
    "    @delegates(ConvLayer.__init__)\n",
    "    def __init__(self, expansion, ni, nh, stride=1, sa=False, sym=False,\n",
    "                 norm_type=NormType.Batch, act_cls=defaults.activation, ndim=2, **kwargs):\n",
    "        super().__init__()\n",
    "        norm2 = NormType.BatchZero if norm_type==NormType.Batch else norm_type\n",
    "        nf,ni = nh*expansion,ni*expansion\n",
    "        layers  = [ConvLayer(ni, nh, 2, stride=stride, norm_type=norm_type, act_cls=None, ndim=ndim, **kwargs),\n",
    "                   ConvLayer(ni, nh, 2, norm_type=norm_type, act_cls=act_cls, ndim=ndim, **kwargs),\n",
    "                   ConvLayer(ni, nh, 2, norm_type=norm_type, act_cls=act_cls, ndim=ndim, **kwargs),\n",
    "                   ConvLayer(nh, nf, 2, norm_type=norm2, act_cls=None, ndim=ndim, **kwargs)\n",
    "        ] if expansion == 1 else [\n",
    "                   ConvLayer(ni, nh, 1, norm_type=norm_type, act_cls=act_cls, ndim=ndim, **kwargs),\n",
    "                   ConvLayer(nh, nh, 2, stride=stride, norm_type=norm_type, act_cls=None, ndim=ndim, **kwargs),\n",
    "                   ConvLayer(nh, nh, 2, norm_type=norm_type, act_cls=act_cls, ndim=ndim, **kwargs),\n",
    "                   ConvLayer(nh, nf, 1, norm_type=norm2, act_cls=None, ndim=ndim, **kwargs)\n",
    "        ]\n",
    "        self.convs = nn.Sequential(*layers)\n",
    "        self.sa = SimpleSelfAttention(nf,ks=1,sym=sym) if sa else noop\n",
    "        self.idconv = noop if ni==nf else ConvLayer(ni, nf, 1, act_cls=None, ndim=ndim, **kwargs)\n",
    "        self.pool = noop if stride==1 else AvgPool(2, ndim=ndim, ceil_mode=True)\n",
    "        self.act = defaults.activation(inplace=True) if act_cls is defaults.activation else act_cls()\n",
    "\n",
    "    def forward(self, x): return self.act(self.sa(self.convs(x)) + self.idconv(self.pool(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class XResNet(nn.Sequential):\n",
    "    def __init__(self, expansion, layers, c_in=3, c_out=1000, sa=False, sym=False, act_cls=defaults.activation):\n",
    "        stem = []\n",
    "        sizes = [c_in,16,32,64] if c_in<3 else [c_in,32,32,64]\n",
    "        for i in range(3):\n",
    "            stem.append(ConvLayer(sizes[i], sizes[i+1], 2, stride=2 if i==0 else 1, act_cls=None))\n",
    "            stem.append(ConvLayer(sizes[i+1], sizes[i+1], 2, act_cls=act_cls))\n",
    "\n",
    "        block_szs = [64//expansion,64,128,256,512] +[256]*(len(layers)-4)\n",
    "        blocks = [self._make_layer(expansion, block_szs[i], block_szs[i+1], l, 1 if i==0 else 2,\n",
    "                                  sa = sa if i==len(layers)-4 else False, sym=sym, act_cls=act_cls)\n",
    "                  for i,l in enumerate(layers)]\n",
    "        super().__init__(\n",
    "            *stem,\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),\n",
    "            *blocks,\n",
    "            nn.AdaptiveAvgPool2d(1), Flatten(),\n",
    "            nn.Linear(block_szs[-1]*expansion, c_out),\n",
    "        )\n",
    "        init_cnn(self)\n",
    "\n",
    "    def _make_layer(self, expansion, ni, nf, blocks, stride, sa, sym, act_cls):\n",
    "        return nn.Sequential(\n",
    "            *[ResBlock(expansion, ni if i==0 else nf, nf, stride if i==0 else 1,\n",
    "                      sa if i==blocks else False, sym=sym, act_cls=act_cls)\n",
    "              for i in range(blocks)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def _xresnet(pretrained, expansion, layers, **kwargs):\n",
    "    # TODO pretrain all sizes. Currently will fail with non-xrn50\n",
    "    url = 'https://s3.amazonaws.com/fast-ai-modelzoo/xrn50_940.pth'\n",
    "    res = XResNet(expansion, layers, **kwargs)\n",
    "    if pretrained: res.load_state_dict(load_state_dict_from_url(url, map_location='cpu')['model'], strict=False)\n",
    "    return res\n",
    "\n",
    "def xresnet18 (pretrained=False, **kwargs): return _xresnet(pretrained, 1, [2, 2,  2, 2], **kwargs)\n",
    "def xresnet34 (pretrained=False, **kwargs): return _xresnet(pretrained, 1, [3, 4,  6, 3], **kwargs)\n",
    "def xresnet50 (pretrained=False, **kwargs): return _xresnet(pretrained, 4, [3, 4,  6, 3], **kwargs)\n",
    "def xresnet101(pretrained=False, **kwargs): return _xresnet(pretrained, 4, [3, 4, 23, 3], **kwargs)\n",
    "def xresnet152(pretrained=False, **kwargs): return _xresnet(pretrained, 4, [3, 8, 36, 3], **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XResNet(\n",
       "  (0): ConvLayer(\n",
       "    (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "    (1): Conv2d(3, 32, kernel_size=(2, 2), stride=(2, 2), bias=False)\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (1): ConvLayer(\n",
       "    (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "    (1): Conv2d(32, 32, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MishJit()\n",
       "  )\n",
       "  (2): ConvLayer(\n",
       "    (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "    (1): Conv2d(32, 32, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (3): ConvLayer(\n",
       "    (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "    (1): Conv2d(32, 32, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MishJit()\n",
       "  )\n",
       "  (4): ConvLayer(\n",
       "    (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "    (1): Conv2d(32, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "    (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (5): ConvLayer(\n",
       "    (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "    (1): Conv2d(64, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "    (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MishJit()\n",
       "  )\n",
       "  (6): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "  (7): Sequential(\n",
       "    (0): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(64, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(64, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (idconv): ConvLayer(\n",
       "        (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (1): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(64, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(64, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (2): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(64, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(64, 64, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "  )\n",
       "  (8): Sequential(\n",
       "    (0): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(2, 2), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (idconv): ConvLayer(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (1): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (2): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (3): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(128, 128, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "  )\n",
       "  (9): Sequential(\n",
       "    (0): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(2, 2), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (idconv): ConvLayer(\n",
       "        (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (1): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (2): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (3): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (4): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (5): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(256, 256, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "  )\n",
       "  (10): Sequential(\n",
       "    (0): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(512, 512, kernel_size=(2, 2), stride=(2, 2), bias=False)\n",
       "          (2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(512, 512, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (idconv): ConvLayer(\n",
       "        (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (1): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(512, 512, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(512, 512, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "    (2): ResBlock(\n",
       "      (convs): Sequential(\n",
       "        (0): ConvLayer(\n",
       "          (0): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (2): MishJit()\n",
       "        )\n",
       "        (1): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(512, 512, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (2): ConvLayer(\n",
       "          (0): ZeroPad2d(padding=(1, 0, 1, 0), value=0.0)\n",
       "          (1): Conv2d(512, 512, kernel_size=(2, 2), stride=(1, 1), bias=False)\n",
       "          (2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          (3): MishJit()\n",
       "        )\n",
       "        (3): ConvLayer(\n",
       "          (0): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (act): MishJit()\n",
       "    )\n",
       "  )\n",
       "  (11): AdaptiveAvgPool2d(output_size=1)\n",
       "  (12): Flatten()\n",
       "  (13): Linear(in_features=2048, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = xresnet50(c_out=dbunch.c, sa=True, sym=False, act_cls=MishJit); m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def opt_func(*args, **kwargs):\n",
    "  opt = RAdam(*args, mom=0.95, wd=1e-2, eps=1e-6, **kwargs)\n",
    "  return Lookahead(opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn = Learner(dbunch, m, LabelSmoothingCrossEntropy(), metrics=[accuracy], opt_func=opt_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.987579</td>\n",
       "      <td>2.458240</td>\n",
       "      <td>0.306000</td>\n",
       "      <td>01:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.719931</td>\n",
       "      <td>2.000820</td>\n",
       "      <td>0.354000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.563770</td>\n",
       "      <td>1.500287</td>\n",
       "      <td>0.544000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.453181</td>\n",
       "      <td>1.432962</td>\n",
       "      <td>0.608000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.262453</td>\n",
       "      <td>1.265259</td>\n",
       "      <td>0.676000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2x2 double no-act 22,306,826 params, 0.66, 0.63, 0.67, 0.63, 0.67 = .65\n",
    "learn.fit_flat_cos(5, 4e-3, pct_start=.72, wd=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.992527</td>\n",
       "      <td>2.297310</td>\n",
       "      <td>0.278000</td>\n",
       "      <td>01:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.791523</td>\n",
       "      <td>2.079929</td>\n",
       "      <td>0.332000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.664257</td>\n",
       "      <td>1.699669</td>\n",
       "      <td>0.444000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.546455</td>\n",
       "      <td>1.567675</td>\n",
       "      <td>0.516000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.362066</td>\n",
       "      <td>1.389904</td>\n",
       "      <td>0.602000</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2x2 double 22,306,826 params, 0.59, 0.6\n",
    "learn.fit_flat_cos(5, 4e-3, pct_start=.72, wd=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.972858</td>\n",
       "      <td>2.023067</td>\n",
       "      <td>0.314000</td>\n",
       "      <td>01:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.791142</td>\n",
       "      <td>2.011371</td>\n",
       "      <td>0.326000</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.635349</td>\n",
       "      <td>1.698493</td>\n",
       "      <td>0.450000</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.543532</td>\n",
       "      <td>1.633707</td>\n",
       "      <td>0.526000</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.355390</td>\n",
       "      <td>1.379606</td>\n",
       "      <td>0.618000</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2x2 same, 0.598, 0.618\n",
    "learn.fit_flat_cos(5, 4e-3, pct_start=.72, wd=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.954447</td>\n",
       "      <td>2.488014</td>\n",
       "      <td>0.268000</td>\n",
       "      <td>01:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.783182</td>\n",
       "      <td>1.990843</td>\n",
       "      <td>0.338000</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.613038</td>\n",
       "      <td>1.576200</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.464347</td>\n",
       "      <td>1.480333</td>\n",
       "      <td>0.560000</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.259610</td>\n",
       "      <td>1.297685</td>\n",
       "      <td>0.648000</td>\n",
       "      <td>00:59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 3x3 23,547,754 params, 0.65, 0.64, 0.66 = 0.65\n",
    "learn.fit_flat_cos(5, 4e-3, pct_start=.72, wd=1e-2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}