train_time_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:50.266800Z",
     "iopub.status.busy": "2021-01-19T05:55:50.266208Z",
     "iopub.status.idle": "2021-01-19T05:55:53.714197Z",
     "shell.execute_reply": "2021-01-19T05:55:53.713082Z"
    },
    "papermill": {
     "duration": 3.468673,
     "end_time": "2021-01-19T05:55:53.714320",
     "exception": false,
     "start_time": "2021-01-19T05:55:50.245647",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path.insert(0,'../input/timm-pytorch-image-models/pytorch-image-models-master')\n",
    "import timm\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "from torch.utils.data.sampler import SubsetRandomSampler, RandomSampler, SequentialSampler\n",
    "import torch.optim as optim\n",
    "from torch.optim.lr_scheduler import CosineAnnealingLR\n",
    "import albumentations as A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.01042,
     "end_time": "2021-01-19T05:55:53.736027",
     "exception": false,
     "start_time": "2021-01-19T05:55:53.725607",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# some hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:53.763593Z",
     "iopub.status.busy": "2021-01-19T05:55:53.761825Z",
     "iopub.status.idle": "2021-01-19T05:55:53.764294Z",
     "shell.execute_reply": "2021-01-19T05:55:53.764720Z"
    },
    "papermill": {
     "duration": 0.018242,
     "end_time": "2021-01-19T05:55:53.764817",
     "exception": false,
     "start_time": "2021-01-19T05:55:53.746575",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_path = '../input/issm2020-ai-challenge/semTrain/semTrain'\n",
    "epoch = 10\n",
    "img_image = 384\n",
    "model_name = 'tf_efficientnet_b4'\n",
    "fold = 5\n",
    "class_num = 10 \n",
    "device = 'cuda'\n",
    "batch_size = 15\n",
    "val_batch_size = 3\n",
    "experience = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.010472,
     "end_time": "2021-01-19T05:55:53.785977",
     "exception": false,
     "start_time": "2021-01-19T05:55:53.775505",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:53.812659Z",
     "iopub.status.busy": "2021-01-19T05:55:53.812049Z",
     "iopub.status.idle": "2021-01-19T05:55:54.583539Z",
     "shell.execute_reply": "2021-01-19T05:55:54.583026Z"
    },
    "papermill": {
     "duration": 0.787071,
     "end_time": "2021-01-19T05:55:54.583643",
     "exception": false,
     "start_time": "2021-01-19T05:55:53.796572",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# make dataset\n",
    "train_data = []\n",
    "for target in os.listdir(train_path):\n",
    "    for image in os.listdir(os.path.join(train_path,target)):\n",
    "        train_data.append([os.path.join(train_path,target,image),int(target)-1])\n",
    "df_train = pd.DataFrame(train_data,columns=['image','target'],index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:54.616538Z",
     "iopub.status.busy": "2021-01-19T05:55:54.615694Z",
     "iopub.status.idle": "2021-01-19T05:55:54.617873Z",
     "shell.execute_reply": "2021-01-19T05:55:54.618280Z"
    },
    "papermill": {
     "duration": 0.022958,
     "end_time": "2021-01-19T05:55:54.618403",
     "exception": false,
     "start_time": "2021-01-19T05:55:54.595445",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create dataset\n",
    "class IssmDataset(Dataset):\n",
    "    def __init__(self, df, mode='train',transform=None):\n",
    "        self.df = df\n",
    "        self.mode = mode\n",
    "        self.transform = transform\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    def __getitem__(self,index):\n",
    "        row = self.df.iloc[index]\n",
    "        img,label = row.image, row.target\n",
    "        img = cv2.imread(img)\n",
    "        img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n",
    "        if self.transform:\n",
    "            img = self.transform(image=img)['image'].astype(np.float32)\n",
    "        else:\n",
    "            img = img.astype(np.float32)\n",
    "        img = img.transpose(2,0,1)\n",
    "        data = torch.tensor(img).float()\n",
    "        if self.mode == 'test':\n",
    "            return data\n",
    "        else:\n",
    "            return data, torch.tensor(label).long()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:54.643996Z",
     "iopub.status.busy": "2021-01-19T05:55:54.643274Z",
     "iopub.status.idle": "2021-01-19T05:55:54.646096Z",
     "shell.execute_reply": "2021-01-19T05:55:54.645690Z"
    },
    "papermill": {
     "duration": 0.016889,
     "end_time": "2021-01-19T05:55:54.646179",
     "exception": false,
     "start_time": "2021-01-19T05:55:54.629290",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# #test\n",
    "# checkset = df_train.reset_index(drop=True)\n",
    "# cdataset = IssmDataset(checkset)\n",
    "# from pylab import rcParams\n",
    "# rcParams['figure.figsize'] = 15,10\n",
    "# for i in range(3):\n",
    "#     _,pos = plt.subplots(1,4)\n",
    "#     for j in range(4):\n",
    "#         idx = np.random.randint(len(cdataset))\n",
    "#         img, label = cdataset[idx]\n",
    "#         # print(img)\n",
    "#         img /= 255\n",
    "#         pos[j].imshow(img.transpose(0,1).transpose(1,2))\n",
    "#         pos[j].set_title(label)\n",
    "#         pos[j].axis('off')\n",
    "#         # plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011374,
     "end_time": "2021-01-19T05:55:54.668523",
     "exception": false,
     "start_time": "2021-01-19T05:55:54.657149",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:54.700301Z",
     "iopub.status.busy": "2021-01-19T05:55:54.699675Z",
     "iopub.status.idle": "2021-01-19T05:55:54.706483Z",
     "shell.execute_reply": "2021-01-19T05:55:54.706948Z"
    },
    "papermill": {
     "duration": 0.027477,
     "end_time": "2021-01-19T05:55:54.707046",
     "exception": false,
     "start_time": "2021-01-19T05:55:54.679569",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/albumentations/augmentations/transforms.py:2611: UserWarning: blur_limit and sigma_limit minimum value can not be both equal to 0. blur_limit minimum value changed to 3.\n",
      "  \"blur_limit and sigma_limit minimum value can not be both equal to 0. \"\n"
     ]
    }
   ],
   "source": [
    "train_transform = A.Compose([\n",
    "    A.VerticalFlip(p=0.5),\n",
    "    A.HorizontalFlip(p=0.5),\n",
    "    A.RandomBrightness(limit=0.05,p=0.5),\n",
    "    A.ShiftScaleRotate(shift_limit=0.1,scale_limit=0.1,p=0.3),\n",
    "    A.OneOf([\n",
    "        A.MotionBlur(blur_limit=5),\n",
    "        A.MedianBlur(blur_limit=5),\n",
    "        A.GaussianBlur(blur_limit=5,sigma_limit=0.1),\n",
    "        A.GaussNoise(var_limit=(5.0, 30.0)),\n",
    "    ], p=0.5),\n",
    "    A.OneOf([\n",
    "        A.OpticalDistortion(distort_limit=1.0),\n",
    "        A.GridDistortion(num_steps=5, distort_limit=1.),\n",
    "        A.ElasticTransform(alpha=3),\n",
    "    ], p=0.5),\n",
    "    A.Resize(img_image,img_image),\n",
    "    A.Cutout(max_h_size=int(img_image * 0.1), max_w_size=int(img_image * 0.1), num_holes=2, p=0.7),    \n",
    "    A.Normalize()\n",
    "    ])\n",
    "\n",
    "test_transform = A.Compose([\n",
    "    A.Resize(img_image,img_image),\n",
    "    A.Normalize()]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:54.733575Z",
     "iopub.status.busy": "2021-01-19T05:55:54.733022Z",
     "iopub.status.idle": "2021-01-19T05:55:54.736963Z",
     "shell.execute_reply": "2021-01-19T05:55:54.736546Z"
    },
    "papermill": {
     "duration": 0.01826,
     "end_time": "2021-01-19T05:55:54.737047",
     "exception": false,
     "start_time": "2021-01-19T05:55:54.718787",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# #test\n",
    "# checkset = df_train.reset_index(drop=True)\n",
    "# cdataset = IssmDataset(checkset,transform=train_transform)\n",
    "# from pylab import rcParams\n",
    "# rcParams['figure.figsize'] = 15,10\n",
    "# for i in range(3):\n",
    "#     _,pos = plt.subplots(1,4)\n",
    "#     for j in range(4):\n",
    "#         idx = np.random.randint(len(cdataset))\n",
    "#         img, label = cdataset[idx]\n",
    "#         # img = cv2.cvtColor(img,cv2.COLOR_RGB2BGR)\n",
    "#         # print(img)\n",
    "#         # img /= 255\n",
    "#         pos[j].imshow(img.transpose(0,1).transpose(1,2))\n",
    "#         pos[j].set_title(label)\n",
    "#         pos[j].axis('off')\n",
    "#         # plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:55:54.767454Z",
     "iopub.status.busy": "2021-01-19T05:55:54.766830Z",
     "iopub.status.idle": "2021-01-19T05:56:04.045292Z",
     "shell.execute_reply": "2021-01-19T05:56:04.044850Z"
    },
    "papermill": {
     "duration": 9.296873,
     "end_time": "2021-01-19T05:56:04.045424",
     "exception": false,
     "start_time": "2021-01-19T05:55:54.748551",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b4_aa-818f208c.pth\" to /root/.cache/torch/hub/checkpoints/tf_efficientnet_b4_aa-818f208c.pth\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "EfficientNet(\n",
       "  (conv_stem): Conv2dSame(3, 48, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
       "  (bn1): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (act1): SiLU(inplace=True)\n",
       "  (blocks): Sequential(\n",
       "    (0): Sequential(\n",
       "      (0): DepthwiseSeparableConv(\n",
       "        (conv_dw): Conv2d(48, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=48, bias=False)\n",
       "        (bn1): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(48, 12, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(12, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pw): Conv2d(48, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(24, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): Identity()\n",
       "      )\n",
       "      (1): DepthwiseSeparableConv(\n",
       "        (conv_dw): Conv2d(24, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=24, bias=False)\n",
       "        (bn1): BatchNorm2d(24, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(24, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(6, 24, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pw): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(24, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): Identity()\n",
       "      )\n",
       "    )\n",
       "    (1): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(144, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(144, 144, kernel_size=(3, 3), stride=(2, 2), groups=144, bias=False)\n",
       "        (bn2): BatchNorm2d(144, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(144, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192, bias=False)\n",
       "        (bn2): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(192, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(8, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192, bias=False)\n",
       "        (bn2): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(192, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(8, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): InvertedResidual(\n",
       "        (conv_pw): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192, bias=False)\n",
       "        (bn2): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(192, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(8, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (2): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(192, 192, kernel_size=(5, 5), stride=(2, 2), groups=192, bias=False)\n",
       "        (bn2): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(192, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(8, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(192, 56, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(56, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(336, 336, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=336, bias=False)\n",
       "        (bn2): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(336, 14, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(14, 336, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(336, 56, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(56, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(336, 336, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=336, bias=False)\n",
       "        (bn2): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(336, 14, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(14, 336, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(336, 56, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(56, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): InvertedResidual(\n",
       "        (conv_pw): Conv2d(56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(336, 336, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=336, bias=False)\n",
       "        (bn2): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(336, 14, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(14, 336, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(336, 56, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(56, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (3): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(336, 336, kernel_size=(3, 3), stride=(2, 2), groups=336, bias=False)\n",
       "        (bn2): BatchNorm2d(336, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(336, 14, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(14, 336, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(336, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (4): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(960, 960, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=960, bias=False)\n",
       "        (bn2): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(960, 40, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(40, 960, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(960, 960, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=960, bias=False)\n",
       "        (bn2): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(960, 40, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(40, 960, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): InvertedResidual(\n",
       "        (conv_pw): Conv2d(160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(960, 960, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=960, bias=False)\n",
       "        (bn2): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(960, 40, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(40, 960, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): InvertedResidual(\n",
       "        (conv_pw): Conv2d(160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(960, 960, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=960, bias=False)\n",
       "        (bn2): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(960, 40, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(40, 960, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): InvertedResidual(\n",
       "        (conv_pw): Conv2d(160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(960, 960, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=960, bias=False)\n",
       "        (bn2): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(960, 40, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(40, 960, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (5): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(960, 960, kernel_size=(5, 5), stride=(2, 2), groups=960, bias=False)\n",
       "        (bn2): BatchNorm2d(960, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(960, 40, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(40, 960, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(960, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (6): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (7): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(272, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (6): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1632, 1632, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1632, bias=False)\n",
       "        (bn2): BatchNorm2d(1632, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1632, 68, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(68, 1632, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1632, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(448, 2688, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(2688, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(2688, 2688, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2688, bias=False)\n",
       "        (bn2): BatchNorm2d(2688, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(2688, 112, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(112, 2688, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(2688, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (conv_head): Conv2d(448, 1792, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "  (bn2): BatchNorm2d(1792, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (act2): SiLU(inplace=True)\n",
       "  (global_pool): SelectAdaptivePool2d (pool_type=avg, flatten=True)\n",
       "  (classifier): Linear(in_features=1792, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_model(model_name, out_features, drop_rate=0.5):\n",
    "    model = timm.create_model(model_name, pretrained=True)\n",
    "    model.drop_rate = drop_rate\n",
    "    model.classifier = nn.Linear(model.classifier.in_features,out_features)\n",
    "    return model\n",
    "\n",
    "eff_model = get_model(model_name,10)\n",
    "eff_model.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:56:04.079375Z",
     "iopub.status.busy": "2021-01-19T05:56:04.078823Z",
     "iopub.status.idle": "2021-01-19T05:56:04.083010Z",
     "shell.execute_reply": "2021-01-19T05:56:04.082552Z"
    },
    "papermill": {
     "duration": 0.024135,
     "end_time": "2021-01-19T05:56:04.083087",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.058952",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class LabelSmoothLoss(nn.Module):\n",
    "    def __init__(self, smoothing=0.0):\n",
    "        super(LabelSmoothLoss, self).__init__()\n",
    "        self.smoothing = smoothing\n",
    "    \n",
    "    def forward(self, input, target):\n",
    "        log_prob = F.log_softmax(input, dim=-1)\n",
    "        weight = input.new_ones(input.size()) * \\\n",
    "            self.smoothing / (input.size(-1) - 1.)\n",
    "        weight.scatter_(-1, target.unsqueeze(-1), (1. - self.smoothing))\n",
    "        loss = (-weight * log_prob).sum(dim=-1).mean()\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:56:04.124117Z",
     "iopub.status.busy": "2021-01-19T05:56:04.122597Z",
     "iopub.status.idle": "2021-01-19T05:56:04.141083Z",
     "shell.execute_reply": "2021-01-19T05:56:04.140595Z"
    },
    "papermill": {
     "duration": 0.044492,
     "end_time": "2021-01-19T05:56:04.141171",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.096679",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(eff_model.parameters(),lr=0.01,momentum=0.9)\n",
    "schduler = CosineAnnealingLR(optimizer, T_max=epoch)\n",
    "# criterion = nn.CrossEntropyLoss() \n",
    "criterion = LabelSmoothLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:56:04.176302Z",
     "iopub.status.busy": "2021-01-19T05:56:04.175790Z",
     "iopub.status.idle": "2021-01-19T05:56:04.179284Z",
     "shell.execute_reply": "2021-01-19T05:56:04.179676Z"
    },
    "papermill": {
     "duration": 0.024495,
     "end_time": "2021-01-19T05:56:04.179775",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.155280",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_one_epoch(model,optimizer,criterion,data_loader):\n",
    "    print('run No.{} epoches, lr = {}'.format(epoch,optimizer.param_groups[0]['lr']))\n",
    "    model.train()\n",
    "    train_loss = []\n",
    "    for data,label in data_loader:\n",
    "        optimizer.zero_grad()\n",
    "        data,label = data.to(device),label.to(device)\n",
    "        pred = model(data)\n",
    "        loss = criterion(pred, label)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        loss_np = loss.detach().cpu().numpy()\n",
    "        train_loss.append(loss_np)\n",
    "        average_loss = sum(train_loss[-100:])/min(len(train_loss),100)\n",
    "        print('average loss = {}, loss = {}'.format(average_loss, loss))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:56:04.214277Z",
     "iopub.status.busy": "2021-01-19T05:56:04.212635Z",
     "iopub.status.idle": "2021-01-19T05:56:04.214994Z",
     "shell.execute_reply": "2021-01-19T05:56:04.215440Z"
    },
    "papermill": {
     "duration": 0.022691,
     "end_time": "2021-01-19T05:56:04.215568",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.192877",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def val_epoch(model,optimizer,criterion,data_loader):\n",
    "    model.eval()\n",
    "    loss_sum = []\n",
    "    with torch.no_grad():\n",
    "        for data,label in data_loader:\n",
    "            data,label = data.to(device),label.to(device)\n",
    "            pred = model(data)\n",
    "            loss = criterion(pred, label)\n",
    "            loss_np = loss.detach().cpu().numpy()\n",
    "            loss_sum.append(loss_np)\n",
    "            average_loss = sum(loss_sum)/len(loss_sum)\n",
    "        print('average loss = {}'.format(average_loss))\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.013397,
     "end_time": "2021-01-19T05:56:04.242418",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.229021",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# cross validation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:56:04.277087Z",
     "iopub.status.busy": "2021-01-19T05:56:04.275378Z",
     "iopub.status.idle": "2021-01-19T05:56:04.277814Z",
     "shell.execute_reply": "2021-01-19T05:56:04.278226Z"
    },
    "papermill": {
     "duration": 0.022573,
     "end_time": "2021-01-19T05:56:04.278334",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.255761",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def preparedata(train_idx,val_idx,df=df_train):\n",
    "    train_=df_train.loc[train_idx,:].reset_index(drop=True)\n",
    "    val_=df_train.loc[val_idx,:].reset_index(drop=True)\n",
    "    train_ds = IssmDataset(train_,'train',train_transform)\n",
    "    val_ds = IssmDataset(val_,'test',test_transform)\n",
    "    train_loader = DataLoader(train_ds,batch_size=batch_size,shuffle=True,num_workers=4)\n",
    "    val_loader = DataLoader(val_ds,batch_size=val_batch_size,shuffle=False,num_workers=4)\n",
    "    return train_loader,val_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-01-19T05:56:04.311132Z",
     "iopub.status.busy": "2021-01-19T05:56:04.310623Z",
     "iopub.status.idle": "2021-01-19T05:56:04.465766Z",
     "shell.execute_reply": "2021-01-19T05:56:04.465102Z"
    },
    "papermill": {
     "duration": 0.173918,
     "end_time": "2021-01-19T05:56:04.465881",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.291963",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "kf = StratifiedKFold(n_splits=fold,shuffle=False)\n",
    "for train_index , test_index in kf.split(df_train.image,df_train.target):\n",
    "    train_loader,val_loader = preparedata(train_index,test_index)\n",
    "    if not experience: \n",
    "        train_one_epoch(eff_model,optimizer,criterion,train_loader)\n",
    "        val_epoch(eff_model,optimizer,criterion,val_loader)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.014368,
     "end_time": "2021-01-19T05:56:04.495586",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.481218",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.013786,
     "end_time": "2021-01-19T05:56:04.523865",
     "exception": false,
     "start_time": "2021-01-19T05:56:04.510079",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 18.562209,
   "end_time": "2021-01-19T05:56:04.946221",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-01-19T05:55:46.384012",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}