Wavenet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VgoLXq8XV4Lg"
   },
   "source": [
    "В **этом ноутбуке** пробуется модель wavenet. Большинство кода взято отсюда https://www.kaggle.com/mobassir/understanding-ion-switching-with-modeling - просто отчаянная попытка хоть как-то повысить скор за счет ноутбуков на каггле."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yT3GqgXwVwFP"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import time\n",
    "import copy\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import KFold, GroupKFold\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "import gc\n",
    "import torchcontrib\n",
    "from tqdm import tqdm\n",
    "from torchcontrib.optim import SWA\n",
    "import torchcontrib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RoOV9gBEWNSi"
   },
   "source": [
    "### 1. Параметры"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3WkybomRWOsN"
   },
   "outputs": [],
   "source": [
    "EPOCHS = 80 #150\n",
    "NNBATCHSIZE = 16\n",
    "GROUP_BATCH_SIZE = 4000\n",
    "SEED = 321\n",
    "LR = 0.001\n",
    "SPLITS = 5\n",
    "\n",
    "outdir = 'wavenet_models'\n",
    "flip = False\n",
    "noise = False\n",
    "\n",
    "\n",
    "if not os.path.exists(outdir):\n",
    "    os.makedirs(outdir)\n",
    "\n",
    "\n",
    "\n",
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    tf.random.set_seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "efibcN0ZWXiS"
   },
   "source": [
    "### 2. Данные"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "a94ikUgBXCVs",
    "outputId": "635973ff-4f6f-4976-8f6e-1d0687269703"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dsH6jKFAWlHh"
   },
   "outputs": [],
   "source": [
    "def read_data():\n",
    "    train = pd.read_csv('/content/gdrive/My Drive/data/train_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})\n",
    "    test  = pd.read_csv('/content/gdrive/My Drive/data/test_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})\n",
    "    sub  = pd.read_csv('/content/gdrive/My Drive/data/sample_submission.csv', dtype={'time': np.float32})\n",
    "    return train, test, sub\n",
    "\n",
    "def batching(df, batch_size):\n",
    "    #print(df)\n",
    "    df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup']).values\n",
    "    df['group'] = df['group'].astype(np.uint16)\n",
    "    return df\n",
    "\n",
    "# normalize the data (standard scaler). We can also try other scalers for a better score!\n",
    "def normalize(train, test):\n",
    "    train_input_mean = train.signal.mean()\n",
    "    train_input_sigma = train.signal.std()\n",
    "    train['signal'] = (train.signal - train_input_mean) / train_input_sigma\n",
    "    test['signal'] = (test.signal - train_input_mean) / train_input_sigma\n",
    "    return train, test\n",
    "\n",
    "# get lead and lags features\n",
    "def lag_with_pct_change(df, windows):\n",
    "    for window in windows:    \n",
    "        df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window).fillna(0)\n",
    "        df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window).fillna(0)\n",
    "    return df\n",
    "\n",
    "# main module to run feature engineering. Here you may want to try and add other features and check if your score imporves :).\n",
    "def run_feat_engineering(df, batch_size):\n",
    "    # create batches\n",
    "    df = batching(df, batch_size = batch_size)\n",
    "    # create leads and lags (1, 2, 3 making them 6 features)\n",
    "    df = lag_with_pct_change(df, [1, 2, 3])\n",
    "    # create signal ** 2 (this is the new feature)\n",
    "    df['signal_2'] = df['signal'] ** 2\n",
    "    return df\n",
    "\n",
    "# fillna with the mean and select features for training\n",
    "def feature_selection(train, test):\n",
    "    features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]\n",
    "    train = train.replace([np.inf, -np.inf], np.nan)\n",
    "    test = test.replace([np.inf, -np.inf], np.nan)\n",
    "    for feature in features:\n",
    "        feature_mean = pd.concat([train[feature], test[feature]], axis = 0).mean()\n",
    "        train[feature] = train[feature].fillna(feature_mean)\n",
    "        test[feature] = test[feature].fillna(feature_mean)\n",
    "    return train, test, features\n",
    "\n",
    "\n",
    "def split(GROUP_BATCH_SIZE=4000, SPLITS=5):\n",
    "    print('Reading Data Started...')\n",
    "    train, test, sample_submission = read_data()\n",
    "    train, test = normalize(train, test)\n",
    "    print('Reading and Normalizing Data Completed')\n",
    "    print('Creating Features')\n",
    "    print('Feature Engineering Started...')\n",
    "    train = run_feat_engineering(train, batch_size=GROUP_BATCH_SIZE)\n",
    "    test = run_feat_engineering(test, batch_size=GROUP_BATCH_SIZE)\n",
    "    train, test, features = feature_selection(train, test)\n",
    "    print(train.head())\n",
    "    print('Feature Engineering Completed...')\n",
    "\n",
    "    target = ['open_channels']\n",
    "    group = train['group']\n",
    "    kf = GroupKFold(n_splits=SPLITS)\n",
    "    splits = [x for x in kf.split(train, train[target], group)]\n",
    "    new_splits = []\n",
    "    for sp in splits:\n",
    "        new_split = []\n",
    "        new_split.append(np.unique(group[sp[0]]))\n",
    "        new_split.append(np.unique(group[sp[1]]))\n",
    "        new_split.append(sp[1])\n",
    "        new_splits.append(new_split)\n",
    "    target_cols = ['open_channels']\n",
    "    print(train.head(), train.shape)\n",
    "    train_tr = np.array(list(train.groupby('group').apply(lambda x: x[target_cols].values))).astype(np.float32)\n",
    "    train = np.array(list(train.groupby('group').apply(lambda x: x[features].values)))\n",
    "    test = np.array(list(test.groupby('group').apply(lambda x: x[features].values)))\n",
    "    print(train.shape, test.shape, train_tr.shape)\n",
    "    return train, test, train_tr, new_splits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p95eDcTXXfSN"
   },
   "source": [
    "### 3. Модель"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UFhMsmLHXuvT"
   },
   "outputs": [],
   "source": [
    "class Wave_Block(nn.Module):\n",
    "    \n",
    "    def __init__(self,in_channels,out_channels,dilation_rates):\n",
    "        super(Wave_Block,self).__init__()\n",
    "        self.num_rates = dilation_rates\n",
    "        self.convs = nn.ModuleList()\n",
    "        self.filter_convs = nn.ModuleList()\n",
    "        self.gate_convs = nn.ModuleList()\n",
    "        \n",
    "        self.convs.append(nn.Conv1d(in_channels,out_channels,kernel_size=1))\n",
    "        dilation_rates = [2**i for i in range(dilation_rates)]\n",
    "        for dilation_rate in dilation_rates:\n",
    "            self.filter_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))\n",
    "            self.gate_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))\n",
    "            self.convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=1))\n",
    "            \n",
    "    def forward(self,x):\n",
    "        x = self.convs[0](x)\n",
    "        res = x\n",
    "        for i in range(self.num_rates):\n",
    "            x = F.tanh(self.filter_convs[i](x))*F.sigmoid(self.gate_convs[i](x))\n",
    "            x = self.convs[i+1](x)\n",
    "            x += res\n",
    "        return x\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "class Classifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        input_size = 128\n",
    "\n",
    "        self.LSTM = nn.GRU(input_size=input_size,hidden_size=64,num_layers=2,batch_first=True,bidirectional=True)\n",
    "        \n",
    "        self.wave_block1 = Wave_Block(8,16,12)\n",
    "        self.wave_block2 = Wave_Block(16,32,8)\n",
    "        self.wave_block3 = Wave_Block(32,64,4)\n",
    "        self.wave_block4 = Wave_Block(64, 128, 1)\n",
    "        self.fc = nn.Linear(128, 11)\n",
    "            \n",
    "    def forward(self,x):\n",
    "        x = x.permute(0, 2, 1)\n",
    "        \n",
    "        x = self.wave_block1(x)\n",
    "        x = self.wave_block2(x)\n",
    "        x = self.wave_block3(x)\n",
    "        \n",
    "        #x,_ = self.LSTM(x)\n",
    "        x = self.wave_block4(x)\n",
    "        x = x.permute(0, 2, 1)\n",
    "        x,_ = self.LSTM(x)\n",
    "        #x = self.conv1(x)\n",
    "        #x = self.attention(x)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "   \n",
    "    \n",
    "class EarlyStopping:\n",
    "    def __init__(self, patience=5, delta=0, checkpoint_path='checkpoint.pt', is_maximize=True):\n",
    "        self.patience, self.delta, self.checkpoint_path = patience, delta, checkpoint_path\n",
    "        self.counter, self.best_score = 0, None\n",
    "        self.is_maximize = is_maximize\n",
    "\n",
    "\n",
    "    def load_best_weights(self, model):\n",
    "        model.load_state_dict(torch.load(self.checkpoint_path))\n",
    "\n",
    "    def __call__(self, score, model):\n",
    "        if self.best_score is None or \\\n",
    "                (score > self.best_score + self.delta if self.is_maximize else score < self.best_score - self.delta):\n",
    "            torch.save(model.state_dict(), self.checkpoint_path)\n",
    "            self.best_score, self.counter = score, 0\n",
    "            return 1\n",
    "        else:\n",
    "            self.counter += 1\n",
    "            if self.counter >= self.patience:\n",
    "                return 2\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6NodcYPJX4eY"
   },
   "source": [
    "### 4. Даталоадеры"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JCFTgLD_X60Z"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "class IronDataset(Dataset):\n",
    "    def __init__(self, data, labels, training=True, transform=None, seq_len=5000, flip=0.5, noise_level=0, class_split=0.0):\n",
    "        self.data = data\n",
    "        self.labels = labels\n",
    "        self.transform = transform\n",
    "        self.training = training\n",
    "        self.flip = flip\n",
    "        self.noise_level = noise_level\n",
    "        self.class_split = class_split\n",
    "        self.seq_len = seq_len\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "\n",
    "        data = self.data[idx]\n",
    "        labels = self.labels[idx]\n",
    "\n",
    "        return [data.astype(np.float32), labels.astype(int)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 386
    },
    "colab_type": "code",
    "id": "luhElQ9zX72g",
    "outputId": "110a4f88-ae0b-4106-f05a-ef020132fe8d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading Data Started...\n",
      "Reading and Normalizing Data Completed\n",
      "Creating Features\n",
      "Feature Engineering Started...\n",
      "     time    signal  ...  signal_shift_neg_3  signal_2\n",
      "0  0.0001 -1.148772  ...           -1.298012  1.319677\n",
      "1  0.0002 -1.184075  ...           -1.303999  1.402034\n",
      "2  0.0003 -1.012891  ...           -1.104036  1.025949\n",
      "3  0.0004 -1.298012  ...           -1.123085  1.684836\n",
      "4  0.0005 -1.303999  ...           -1.082287  1.700413\n",
      "\n",
      "[5 rows x 11 columns]\n",
      "Feature Engineering Completed...\n",
      "     time    signal  ...  signal_shift_neg_3  signal_2\n",
      "0  0.0001 -1.148772  ...           -1.298012  1.319677\n",
      "1  0.0002 -1.184075  ...           -1.303999  1.402034\n",
      "2  0.0003 -1.012891  ...           -1.104036  1.025949\n",
      "3  0.0004 -1.298012  ...           -1.123085  1.684836\n",
      "4  0.0005 -1.303999  ...           -1.082287  1.700413\n",
      "\n",
      "[5 rows x 11 columns] (5000000, 11)\n",
      "(1250, 4000, 8) (500, 4000, 8) (1250, 4000, 1)\n"
     ]
    }
   ],
   "source": [
    "train, test, train_tr, new_splits = split()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YHmA3yfpY7PS"
   },
   "source": [
    "### 5. Обучение"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "46_4WPgKX-hR",
    "outputId": "b18e1187-d07e-48a8-faed-e97dcf0f5223"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Classifier(\n",
       "  (LSTM): GRU(128, 64, num_layers=2, batch_first=True, bidirectional=True)\n",
       "  (wave_block1): Wave_Block(\n",
       "    (convs): ModuleList(\n",
       "      (0): Conv1d(8, 16, kernel_size=(1,), stride=(1,))\n",
       "      (1): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (2): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (3): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (4): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (5): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (6): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (7): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (8): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (9): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (10): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (11): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "      (12): Conv1d(16, 16, kernel_size=(1,), stride=(1,))\n",
       "    )\n",
       "    (filter_convs): ModuleList(\n",
       "      (0): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "      (1): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,))\n",
       "      (2): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,))\n",
       "      (3): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,))\n",
       "      (4): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,))\n",
       "      (5): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,))\n",
       "      (6): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,))\n",
       "      (7): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,))\n",
       "      (8): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(256,), dilation=(256,))\n",
       "      (9): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(512,), dilation=(512,))\n",
       "      (10): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1024,), dilation=(1024,))\n",
       "      (11): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(2048,), dilation=(2048,))\n",
       "    )\n",
       "    (gate_convs): ModuleList(\n",
       "      (0): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "      (1): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,))\n",
       "      (2): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,))\n",
       "      (3): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,))\n",
       "      (4): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,))\n",
       "      (5): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,))\n",
       "      (6): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,))\n",
       "      (7): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,))\n",
       "      (8): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(256,), dilation=(256,))\n",
       "      (9): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(512,), dilation=(512,))\n",
       "      (10): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1024,), dilation=(1024,))\n",
       "      (11): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(2048,), dilation=(2048,))\n",
       "    )\n",
       "  )\n",
       "  (wave_block2): Wave_Block(\n",
       "    (convs): ModuleList(\n",
       "      (0): Conv1d(16, 32, kernel_size=(1,), stride=(1,))\n",
       "      (1): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "      (2): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "      (3): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "      (4): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "      (5): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "      (6): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "      (7): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "      (8): Conv1d(32, 32, kernel_size=(1,), stride=(1,))\n",
       "    )\n",
       "    (filter_convs): ModuleList(\n",
       "      (0): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "      (1): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,))\n",
       "      (2): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,))\n",
       "      (3): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,))\n",
       "      (4): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,))\n",
       "      (5): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,))\n",
       "      (6): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,))\n",
       "      (7): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,))\n",
       "    )\n",
       "    (gate_convs): ModuleList(\n",
       "      (0): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "      (1): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,))\n",
       "      (2): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,))\n",
       "      (3): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,))\n",
       "      (4): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(16,), dilation=(16,))\n",
       "      (5): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(32,), dilation=(32,))\n",
       "      (6): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(64,), dilation=(64,))\n",
       "      (7): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(128,), dilation=(128,))\n",
       "    )\n",
       "  )\n",
       "  (wave_block3): Wave_Block(\n",
       "    (convs): ModuleList(\n",
       "      (0): Conv1d(32, 64, kernel_size=(1,), stride=(1,))\n",
       "      (1): Conv1d(64, 64, kernel_size=(1,), stride=(1,))\n",
       "      (2): Conv1d(64, 64, kernel_size=(1,), stride=(1,))\n",
       "      (3): Conv1d(64, 64, kernel_size=(1,), stride=(1,))\n",
       "      (4): Conv1d(64, 64, kernel_size=(1,), stride=(1,))\n",
       "    )\n",
       "    (filter_convs): ModuleList(\n",
       "      (0): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "      (1): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,))\n",
       "      (2): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,))\n",
       "      (3): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,))\n",
       "    )\n",
       "    (gate_convs): ModuleList(\n",
       "      (0): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "      (1): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(2,), dilation=(2,))\n",
       "      (2): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(4,), dilation=(4,))\n",
       "      (3): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(8,), dilation=(8,))\n",
       "    )\n",
       "  )\n",
       "  (wave_block4): Wave_Block(\n",
       "    (convs): ModuleList(\n",
       "      (0): Conv1d(64, 128, kernel_size=(1,), stride=(1,))\n",
       "      (1): Conv1d(128, 128, kernel_size=(1,), stride=(1,))\n",
       "    )\n",
       "    (filter_convs): ModuleList(\n",
       "      (0): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "    )\n",
       "    (gate_convs): ModuleList(\n",
       "      (0): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "    )\n",
       "  )\n",
       "  (fc): Linear(in_features=128, out_features=11, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Classifier()\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "XRNDJ6uMYLmW",
    "outputId": "a5491d62-3a85-418c-d81e-e0974ba49eac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold : 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**********************************\n",
      "Folder : 0 Epoch : 0\n",
      "Curr learning_rate: 0.001000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1558: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
      "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n",
      "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1569: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n",
      "100%|██████████| 63/63 [00:35<00:00,  1.78it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.870355, valid_loss: 0.605278\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.515272, valid_f1: 0.622803\n",
      "save folder 0 global val max f1 model score 0.622803\n",
      "**********************************\n",
      "Folder : 0 Epoch : 1\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.349414, valid_loss: 0.328683\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.747272, valid_f1: 0.778421\n",
      "save folder 0 global val max f1 model score 0.778421\n",
      "**********************************\n",
      "Folder : 0 Epoch : 2\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.78it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.209879, valid_loss: 0.234360\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.813873, valid_f1: 0.795859\n",
      "save folder 0 global val max f1 model score 0.795859\n",
      "**********************************\n",
      "Folder : 0 Epoch : 3\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.78it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.146862, valid_loss: 0.158649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.856624, valid_f1: 0.840171\n",
      "save folder 0 global val max f1 model score 0.840171\n",
      "**********************************\n",
      "Folder : 0 Epoch : 4\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.78it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.123236, valid_loss: 0.123807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.904361, valid_f1: 0.923503\n",
      "save folder 0 global val max f1 model score 0.923503\n",
      "**********************************\n",
      "Folder : 0 Epoch : 5\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.106862, valid_loss: 0.117816\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.926388, valid_f1: 0.922801\n",
      "**********************************\n",
      "Folder : 0 Epoch : 6\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.78it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.103864, valid_loss: 0.144265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.928481, valid_f1: 0.894290\n",
      "**********************************\n",
      "Folder : 0 Epoch : 7\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.154370, valid_loss: 0.104685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.880300, valid_f1: 0.934297\n",
      "save folder 0 global val max f1 model score 0.934297\n",
      "**********************************\n",
      "Folder : 0 Epoch : 8\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092779, valid_loss: 0.112344\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934207, valid_f1: 0.928698\n",
      "**********************************\n",
      "Folder : 0 Epoch : 9\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092063, valid_loss: 0.099416\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934900, valid_f1: 0.934660\n",
      "save folder 0 global val max f1 model score 0.934660\n",
      "**********************************\n",
      "Folder : 0 Epoch : 10\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088239, valid_loss: 0.098133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936298, valid_f1: 0.936065\n",
      "save folder 0 global val max f1 model score 0.936065\n",
      "**********************************\n",
      "Folder : 0 Epoch : 11\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087837, valid_loss: 0.095972\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935934, valid_f1: 0.936748\n",
      "save folder 0 global val max f1 model score 0.936748\n",
      "**********************************\n",
      "Folder : 0 Epoch : 12\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.090313, valid_loss: 0.097000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932941, valid_f1: 0.935618\n",
      "**********************************\n",
      "Folder : 0 Epoch : 13\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085793, valid_loss: 0.095936\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936644, valid_f1: 0.936594\n",
      "**********************************\n",
      "Folder : 0 Epoch : 14\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085082, valid_loss: 0.094610\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937157, valid_f1: 0.936067\n",
      "**********************************\n",
      "Folder : 0 Epoch : 15\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085113, valid_loss: 0.094261\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936143, valid_f1: 0.937158\n",
      "save folder 0 global val max f1 model score 0.937158\n",
      "**********************************\n",
      "Folder : 0 Epoch : 16\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084764, valid_loss: 0.093377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936944, valid_f1: 0.937073\n",
      "**********************************\n",
      "Folder : 0 Epoch : 17\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085975, valid_loss: 0.093490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935243, valid_f1: 0.937195\n",
      "save folder 0 global val max f1 model score 0.937195\n",
      "**********************************\n",
      "Folder : 0 Epoch : 18\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084430, valid_loss: 0.092541\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936250, valid_f1: 0.937565\n",
      "save folder 0 global val max f1 model score 0.937565\n",
      "**********************************\n",
      "Folder : 0 Epoch : 19\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084028, valid_loss: 0.093503\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937094, valid_f1: 0.936073\n",
      "**********************************\n",
      "Folder : 0 Epoch : 20\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083804, valid_loss: 0.092146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936711, valid_f1: 0.937721\n",
      "save folder 0 global val max f1 model score 0.937721\n",
      "**********************************\n",
      "Folder : 0 Epoch : 21\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093798, valid_loss: 0.092079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.928292, valid_f1: 0.937321\n",
      "**********************************\n",
      "Folder : 0 Epoch : 22\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.090002, valid_loss: 0.103792\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.930156, valid_f1: 0.930012\n",
      "**********************************\n",
      "Folder : 0 Epoch : 23\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085812, valid_loss: 0.092047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935018, valid_f1: 0.937077\n",
      "**********************************\n",
      "Folder : 0 Epoch : 24\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082583, valid_loss: 0.095502\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937426, valid_f1: 0.934456\n",
      "**********************************\n",
      "Folder : 0 Epoch : 25\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083827, valid_loss: 0.092129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936633, valid_f1: 0.936624\n",
      "**********************************\n",
      "Folder : 0 Epoch : 26\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084708, valid_loss: 0.091554\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935793, valid_f1: 0.936663\n",
      "**********************************\n",
      "Folder : 0 Epoch : 27\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081994, valid_loss: 0.091182\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937541, valid_f1: 0.937807\n",
      "save folder 0 global val max f1 model score 0.937807\n",
      "**********************************\n",
      "Folder : 0 Epoch : 28\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082770, valid_loss: 0.091020\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937385, valid_f1: 0.937834\n",
      "save folder 0 global val max f1 model score 0.937834\n",
      "**********************************\n",
      "Folder : 0 Epoch : 29\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083206, valid_loss: 0.093901\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936882, valid_f1: 0.936716\n",
      "**********************************\n",
      "Folder : 0 Epoch : 30\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083637, valid_loss: 0.093427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936057, valid_f1: 0.936953\n",
      "**********************************\n",
      "Folder : 0 Epoch : 31\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086184, valid_loss: 0.095751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934614, valid_f1: 0.934838\n",
      "**********************************\n",
      "Folder : 0 Epoch : 32\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089964, valid_loss: 0.091970\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.930007, valid_f1: 0.934982\n",
      "**********************************\n",
      "Folder : 0 Epoch : 33\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081939, valid_loss: 0.091605\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937475, valid_f1: 0.937262\n",
      "**********************************\n",
      "Folder : 0 Epoch : 34\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081814, valid_loss: 0.090345\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937651, valid_f1: 0.938114\n",
      "save folder 0 global val max f1 model score 0.938114\n",
      "**********************************\n",
      "Folder : 0 Epoch : 35\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081443, valid_loss: 0.096056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937900, valid_f1: 0.932444\n",
      "**********************************\n",
      "Folder : 0 Epoch : 36\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086314, valid_loss: 0.090035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934179, valid_f1: 0.937804\n",
      "**********************************\n",
      "Folder : 0 Epoch : 37\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.137177, valid_loss: 0.092168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.901158, valid_f1: 0.936398\n",
      "**********************************\n",
      "Folder : 0 Epoch : 38\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083875, valid_loss: 0.123052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935161, valid_f1: 0.912411\n",
      "**********************************\n",
      "Folder : 0 Epoch : 39\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086033, valid_loss: 0.089880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934947, valid_f1: 0.938428\n",
      "save folder 0 global val max f1 model score 0.938428\n",
      "**********************************\n",
      "Folder : 0 Epoch : 40\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081740, valid_loss: 0.091826\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937333, valid_f1: 0.937464\n",
      "**********************************\n",
      "Folder : 0 Epoch : 41\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081730, valid_loss: 0.090248\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937748, valid_f1: 0.937971\n",
      "**********************************\n",
      "Folder : 0 Epoch : 42\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079979, valid_loss: 0.090795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938163, valid_f1: 0.938013\n",
      "**********************************\n",
      "Folder : 0 Epoch : 43\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081895, valid_loss: 0.089993\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937117, valid_f1: 0.938289\n",
      "**********************************\n",
      "Folder : 0 Epoch : 44\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080036, valid_loss: 0.090362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938719, valid_f1: 0.937737\n",
      "**********************************\n",
      "Folder : 0 Epoch : 45\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080393, valid_loss: 0.089583\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938121, valid_f1: 0.938459\n",
      "save folder 0 global val max f1 model score 0.938459\n",
      "**********************************\n",
      "Folder : 0 Epoch : 46\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079683, valid_loss: 0.090795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938796, valid_f1: 0.937705\n",
      "**********************************\n",
      "Folder : 0 Epoch : 47\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080790, valid_loss: 0.093539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938154, valid_f1: 0.933011\n",
      "**********************************\n",
      "Folder : 0 Epoch : 48\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080783, valid_loss: 0.089836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937909, valid_f1: 0.938345\n",
      "**********************************\n",
      "Folder : 0 Epoch : 49\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080938, valid_loss: 0.090663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938058, valid_f1: 0.937922\n",
      "**********************************\n",
      "Folder : 0 Epoch : 50\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081489, valid_loss: 0.090729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936975, valid_f1: 0.937910\n",
      "**********************************\n",
      "Folder : 0 Epoch : 51\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081023, valid_loss: 0.090258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938347, valid_f1: 0.937181\n",
      "**********************************\n",
      "Folder : 0 Epoch : 52\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080548, valid_loss: 0.090517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938382, valid_f1: 0.937684\n",
      "**********************************\n",
      "Folder : 0 Epoch : 53\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080171, valid_loss: 0.089503\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938345, valid_f1: 0.938465\n",
      "save folder 0 global val max f1 model score 0.938465\n",
      "**********************************\n",
      "Folder : 0 Epoch : 54\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080297, valid_loss: 0.089457\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938945, valid_f1: 0.937825\n",
      "**********************************\n",
      "Folder : 0 Epoch : 55\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080217, valid_loss: 0.090426\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937921, valid_f1: 0.938259\n",
      "**********************************\n",
      "Folder : 0 Epoch : 56\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079462, valid_loss: 0.091018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938352, valid_f1: 0.936477\n",
      "**********************************\n",
      "Folder : 0 Epoch : 57\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080585, valid_loss: 0.091366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937555, valid_f1: 0.937110\n",
      "**********************************\n",
      "Folder : 0 Epoch : 58\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080552, valid_loss: 0.090043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938013, valid_f1: 0.937521\n",
      "**********************************\n",
      "Folder : 0 Epoch : 59\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079595, valid_loss: 0.090873\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938442, valid_f1: 0.936823\n",
      "**********************************\n",
      "Folder : 0 Epoch : 60\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080736, valid_loss: 0.089542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937802, valid_f1: 0.938117\n",
      "**********************************\n",
      "Folder : 0 Epoch : 61\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.078987, valid_loss: 0.091237\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938886, valid_f1: 0.936921\n",
      "**********************************\n",
      "Folder : 0 Epoch : 62\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.78it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081170, valid_loss: 0.090120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936710, valid_f1: 0.936715\n",
      "**********************************\n",
      "Folder : 0 Epoch : 63\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080194, valid_loss: 0.090195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938328, valid_f1: 0.938122\n",
      "**********************************\n",
      "Folder : 0 Epoch : 64\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079874, valid_loss: 0.089353\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938302, valid_f1: 0.938216\n",
      "**********************************\n",
      "Folder : 0 Epoch : 65\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080015, valid_loss: 0.093042\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938486, valid_f1: 0.934178\n",
      "**********************************\n",
      "Folder : 0 Epoch : 66\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083872, valid_loss: 0.090196\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935308, valid_f1: 0.937574\n",
      "**********************************\n",
      "Folder : 0 Epoch : 67\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080961, valid_loss: 0.090207\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938008, valid_f1: 0.937694\n",
      "**********************************\n",
      "Folder : 0 Epoch : 68\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081897, valid_loss: 0.091264\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936765, valid_f1: 0.937401\n",
      "**********************************\n",
      "Folder : 0 Epoch : 69\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080860, valid_loss: 0.111648\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937889, valid_f1: 0.924090\n",
      "**********************************\n",
      "Folder : 0 Epoch : 70\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082129, valid_loss: 0.089613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936407, valid_f1: 0.937956\n",
      "**********************************\n",
      "Folder : 0 Epoch : 71\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079753, valid_loss: 0.091627\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938477, valid_f1: 0.937121\n",
      "**********************************\n",
      "Folder : 0 Epoch : 72\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079372, valid_loss: 0.090862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938434, valid_f1: 0.937704\n",
      "**********************************\n",
      "Folder : 0 Epoch : 73\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.078702, valid_loss: 0.090078\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939107, valid_f1: 0.938060\n",
      "**********************************\n",
      "Folder : 0 Epoch : 74\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.078876, valid_loss: 0.090796\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938856, valid_f1: 0.936893\n",
      "**********************************\n",
      "Folder : 0 Epoch : 75\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080971, valid_loss: 0.089843\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937070, valid_f1: 0.938335\n",
      "**********************************\n",
      "Folder : 0 Epoch : 76\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080389, valid_loss: 0.091926\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937691, valid_f1: 0.936708\n",
      "**********************************\n",
      "Folder : 0 Epoch : 77\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079313, valid_loss: 0.090326\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938472, valid_f1: 0.937571\n",
      "**********************************\n",
      "Folder : 0 Epoch : 78\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.078886, valid_loss: 0.090125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939045, valid_f1: 0.938053\n",
      "**********************************\n",
      "Folder : 0 Epoch : 79\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081059, valid_loss: 0.091097\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/32 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936448, valid_f1: 0.936769\n",
      "Folder 0 finally best global max f1 score is 0.9384645053011877\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:07<00:00,  4.29it/s]\n",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold : 1\n",
      "**********************************\n",
      "Folder : 1 Epoch : 0\n",
      "Curr learning_rate: 0.001000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 1.046662, valid_loss: 0.677108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.411577, valid_f1: 0.540173\n",
      "save folder 1 global val max f1 model score 0.540173\n",
      "**********************************\n",
      "Folder : 1 Epoch : 1\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.395861, valid_loss: 0.407072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.739909, valid_f1: 0.759746\n",
      "save folder 1 global val max f1 model score 0.759746\n",
      "**********************************\n",
      "Folder : 1 Epoch : 2\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.241965, valid_loss: 0.231281\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.809051, valid_f1: 0.798607\n",
      "save folder 1 global val max f1 model score 0.798607\n",
      "**********************************\n",
      "Folder : 1 Epoch : 3\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.168693, valid_loss: 0.151215\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.826290, valid_f1: 0.837548\n",
      "save folder 1 global val max f1 model score 0.837548\n",
      "**********************************\n",
      "Folder : 1 Epoch : 4\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.136644, valid_loss: 0.128245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.844110, valid_f1: 0.841584\n",
      "save folder 1 global val max f1 model score 0.841584\n",
      "**********************************\n",
      "Folder : 1 Epoch : 5\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.118516, valid_loss: 0.115176\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.893488, valid_f1: 0.902031\n",
      "save folder 1 global val max f1 model score 0.902031\n",
      "**********************************\n",
      "Folder : 1 Epoch : 6\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.121879, valid_loss: 0.116778\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.910328, valid_f1: 0.927805\n",
      "save folder 1 global val max f1 model score 0.927805\n",
      "**********************************\n",
      "Folder : 1 Epoch : 7\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.124165, valid_loss: 0.121117\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.910755, valid_f1: 0.925671\n",
      "**********************************\n",
      "Folder : 1 Epoch : 8\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.098085, valid_loss: 0.100063\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933922, valid_f1: 0.934386\n",
      "save folder 1 global val max f1 model score 0.934386\n",
      "**********************************\n",
      "Folder : 1 Epoch : 9\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.095394, valid_loss: 0.092822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934087, valid_f1: 0.937277\n",
      "save folder 1 global val max f1 model score 0.937277\n",
      "**********************************\n",
      "Folder : 1 Epoch : 10\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.095640, valid_loss: 0.092326\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931234, valid_f1: 0.936881\n",
      "**********************************\n",
      "Folder : 1 Epoch : 11\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.095922, valid_loss: 0.099680\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.930723, valid_f1: 0.931670\n",
      "**********************************\n",
      "Folder : 1 Epoch : 12\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.091365, valid_loss: 0.089807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934753, valid_f1: 0.937389\n",
      "save folder 1 global val max f1 model score 0.937389\n",
      "**********************************\n",
      "Folder : 1 Epoch : 13\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089469, valid_loss: 0.089128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935846, valid_f1: 0.938371\n",
      "save folder 1 global val max f1 model score 0.938371\n",
      "**********************************\n",
      "Folder : 1 Epoch : 14\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089425, valid_loss: 0.088231\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935030, valid_f1: 0.938093\n",
      "**********************************\n",
      "Folder : 1 Epoch : 15\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093277, valid_loss: 0.091655\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931862, valid_f1: 0.933630\n",
      "**********************************\n",
      "Folder : 1 Epoch : 16\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.090937, valid_loss: 0.097456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932617, valid_f1: 0.927226\n",
      "**********************************\n",
      "Folder : 1 Epoch : 17\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088315, valid_loss: 0.086735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935880, valid_f1: 0.938758\n",
      "save folder 1 global val max f1 model score 0.938758\n",
      "**********************************\n",
      "Folder : 1 Epoch : 18\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092138, valid_loss: 0.088035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931050, valid_f1: 0.937716\n",
      "**********************************\n",
      "Folder : 1 Epoch : 19\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.091130, valid_loss: 0.088737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931957, valid_f1: 0.937869\n",
      "**********************************\n",
      "Folder : 1 Epoch : 20\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.145688, valid_loss: 0.087807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.909104, valid_f1: 0.936605\n",
      "**********************************\n",
      "Folder : 1 Epoch : 21\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093504, valid_loss: 0.189222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.929224, valid_f1: 0.888646\n",
      "**********************************\n",
      "Folder : 1 Epoch : 22\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.100613, valid_loss: 0.086886\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.927270, valid_f1: 0.937843\n",
      "**********************************\n",
      "Folder : 1 Epoch : 23\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085371, valid_loss: 0.087267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937116, valid_f1: 0.938295\n",
      "**********************************\n",
      "Folder : 1 Epoch : 24\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085658, valid_loss: 0.085629\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937034, valid_f1: 0.938991\n",
      "save folder 1 global val max f1 model score 0.938991\n",
      "**********************************\n",
      "Folder : 1 Epoch : 25\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084275, valid_loss: 0.085852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937430, valid_f1: 0.938815\n",
      "**********************************\n",
      "Folder : 1 Epoch : 26\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084593, valid_loss: 0.084676\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937423, valid_f1: 0.939317\n",
      "save folder 1 global val max f1 model score 0.939317\n",
      "**********************************\n",
      "Folder : 1 Epoch : 27\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084743, valid_loss: 0.085510\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937611, valid_f1: 0.938954\n",
      "**********************************\n",
      "Folder : 1 Epoch : 28\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084267, valid_loss: 0.084548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936686, valid_f1: 0.939349\n",
      "save folder 1 global val max f1 model score 0.939349\n",
      "**********************************\n",
      "Folder : 1 Epoch : 29\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086743, valid_loss: 0.087139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933803, valid_f1: 0.933908\n",
      "**********************************\n",
      "Folder : 1 Epoch : 30\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084726, valid_loss: 0.091292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936677, valid_f1: 0.933816\n",
      "**********************************\n",
      "Folder : 1 Epoch : 31\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087240, valid_loss: 0.084731\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934229, valid_f1: 0.938280\n",
      "**********************************\n",
      "Folder : 1 Epoch : 32\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089632, valid_loss: 0.084692\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932663, valid_f1: 0.938035\n",
      "**********************************\n",
      "Folder : 1 Epoch : 33\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083694, valid_loss: 0.085292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937834, valid_f1: 0.938835\n",
      "**********************************\n",
      "Folder : 1 Epoch : 34\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083944, valid_loss: 0.083901\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937258, valid_f1: 0.938788\n",
      "**********************************\n",
      "Folder : 1 Epoch : 35\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083717, valid_loss: 0.084546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937306, valid_f1: 0.938036\n",
      "**********************************\n",
      "Folder : 1 Epoch : 36\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083907, valid_loss: 0.084501\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937048, valid_f1: 0.939120\n",
      "**********************************\n",
      "Folder : 1 Epoch : 37\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085226, valid_loss: 0.085039\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935507, valid_f1: 0.937528\n",
      "**********************************\n",
      "Folder : 1 Epoch : 38\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085456, valid_loss: 0.085058\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935567, valid_f1: 0.937553\n",
      "**********************************\n",
      "Folder : 1 Epoch : 39\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085018, valid_loss: 0.084589\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935966, valid_f1: 0.938047\n",
      "**********************************\n",
      "Folder : 1 Epoch : 40\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084360, valid_loss: 0.084680\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936504, valid_f1: 0.937867\n",
      "**********************************\n",
      "Folder : 1 Epoch : 41\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085410, valid_loss: 0.083999\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934728, valid_f1: 0.939368\n",
      "save folder 1 global val max f1 model score 0.939368\n",
      "**********************************\n",
      "Folder : 1 Epoch : 42\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084568, valid_loss: 0.084357\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935969, valid_f1: 0.936712\n",
      "**********************************\n",
      "Folder : 1 Epoch : 43\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085214, valid_loss: 0.086712\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935861, valid_f1: 0.937896\n",
      "**********************************\n",
      "Folder : 1 Epoch : 44\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.097498, valid_loss: 0.083935\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.925273, valid_f1: 0.938608\n",
      "**********************************\n",
      "Folder : 1 Epoch : 45\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082440, valid_loss: 0.089712\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937891, valid_f1: 0.935460\n",
      "**********************************\n",
      "Folder : 1 Epoch : 46\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083382, valid_loss: 0.083405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937010, valid_f1: 0.939570\n",
      "save folder 1 global val max f1 model score 0.939570\n",
      "**********************************\n",
      "Folder : 1 Epoch : 47\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082086, valid_loss: 0.083767\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938172, valid_f1: 0.938432\n",
      "**********************************\n",
      "Folder : 1 Epoch : 48\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082799, valid_loss: 0.083141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937912, valid_f1: 0.939065\n",
      "**********************************\n",
      "Folder : 1 Epoch : 49\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082561, valid_loss: 0.084330\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937859, valid_f1: 0.938958\n",
      "**********************************\n",
      "Folder : 1 Epoch : 50\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082761, valid_loss: 0.085545\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937906, valid_f1: 0.938109\n",
      "**********************************\n",
      "Folder : 1 Epoch : 51\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093417, valid_loss: 0.083563\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.926937, valid_f1: 0.939054\n",
      "**********************************\n",
      "Folder : 1 Epoch : 52\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081990, valid_loss: 0.087288\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937903, valid_f1: 0.936793\n",
      "**********************************\n",
      "Folder : 1 Epoch : 53\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083176, valid_loss: 0.083063\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937573, valid_f1: 0.939219\n",
      "**********************************\n",
      "Folder : 1 Epoch : 54\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082022, valid_loss: 0.083490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937859, valid_f1: 0.938806\n",
      "**********************************\n",
      "Folder : 1 Epoch : 55\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081295, valid_loss: 0.084233\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938203, valid_f1: 0.938833\n",
      "**********************************\n",
      "Folder : 1 Epoch : 56\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083489, valid_loss: 0.083613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936768, valid_f1: 0.939014\n",
      "**********************************\n",
      "Folder : 1 Epoch : 57\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081918, valid_loss: 0.083395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938239, valid_f1: 0.939175\n",
      "**********************************\n",
      "Folder : 1 Epoch : 58\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082281, valid_loss: 0.083357\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938223, valid_f1: 0.939494\n",
      "**********************************\n",
      "Folder : 1 Epoch : 59\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081664, valid_loss: 0.083394\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938162, valid_f1: 0.939557\n",
      "**********************************\n",
      "Folder : 1 Epoch : 60\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082304, valid_loss: 0.083105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937715, valid_f1: 0.939586\n",
      "save folder 1 global val max f1 model score 0.939586\n",
      "**********************************\n",
      "Folder : 1 Epoch : 61\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082363, valid_loss: 0.083751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937304, valid_f1: 0.939482\n",
      "**********************************\n",
      "Folder : 1 Epoch : 62\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084154, valid_loss: 0.082736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936350, valid_f1: 0.939206\n",
      "**********************************\n",
      "Folder : 1 Epoch : 63\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081644, valid_loss: 0.083487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938052, valid_f1: 0.938750\n",
      "**********************************\n",
      "Folder : 1 Epoch : 64\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081842, valid_loss: 0.083402\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938326, valid_f1: 0.939486\n",
      "**********************************\n",
      "Folder : 1 Epoch : 65\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082352, valid_loss: 0.083054\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937215, valid_f1: 0.939393\n",
      "**********************************\n",
      "Folder : 1 Epoch : 66\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083145, valid_loss: 0.083636\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936226, valid_f1: 0.939147\n",
      "**********************************\n",
      "Folder : 1 Epoch : 67\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085796, valid_loss: 0.083191\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935103, valid_f1: 0.939641\n",
      "save folder 1 global val max f1 model score 0.939641\n",
      "**********************************\n",
      "Folder : 1 Epoch : 68\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086294, valid_loss: 0.087522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933532, valid_f1: 0.938148\n",
      "**********************************\n",
      "Folder : 1 Epoch : 69\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082617, valid_loss: 0.083538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937421, valid_f1: 0.939662\n",
      "save folder 1 global val max f1 model score 0.939662\n",
      "**********************************\n",
      "Folder : 1 Epoch : 70\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082472, valid_loss: 0.083318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937750, valid_f1: 0.939388\n",
      "**********************************\n",
      "Folder : 1 Epoch : 71\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081807, valid_loss: 0.083565\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937819, valid_f1: 0.939654\n",
      "**********************************\n",
      "Folder : 1 Epoch : 72\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081168, valid_loss: 0.086538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938406, valid_f1: 0.936319\n",
      "**********************************\n",
      "Folder : 1 Epoch : 73\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089350, valid_loss: 0.083748\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931021, valid_f1: 0.939506\n",
      "**********************************\n",
      "Folder : 1 Epoch : 74\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085026, valid_loss: 0.086165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934984, valid_f1: 0.938219\n",
      "**********************************\n",
      "Folder : 1 Epoch : 75\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083336, valid_loss: 0.084276\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936929, valid_f1: 0.939221\n",
      "**********************************\n",
      "Folder : 1 Epoch : 76\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081596, valid_loss: 0.082902\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937719, valid_f1: 0.939280\n",
      "**********************************\n",
      "Folder : 1 Epoch : 77\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080422, valid_loss: 0.087307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938821, valid_f1: 0.938456\n",
      "**********************************\n",
      "Folder : 1 Epoch : 78\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081947, valid_loss: 0.082127\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938128, valid_f1: 0.939936\n",
      "save folder 1 global val max f1 model score 0.939936\n",
      "**********************************\n",
      "Folder : 1 Epoch : 79\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081090, valid_loss: 0.091261\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/32 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938146, valid_f1: 0.933940\n",
      "Folder 1 finally best global max f1 score is 0.9399362453324153\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:07<00:00,  4.26it/s]\n",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold : 2\n",
      "**********************************\n",
      "Folder : 2 Epoch : 0\n",
      "Curr learning_rate: 0.001000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.888532, valid_loss: 0.611935\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.469686, valid_f1: 0.540450\n",
      "save folder 2 global val max f1 model score 0.540450\n",
      "**********************************\n",
      "Folder : 2 Epoch : 1\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.352293, valid_loss: 0.367596\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.738649, valid_f1: 0.715353\n",
      "save folder 2 global val max f1 model score 0.715353\n",
      "**********************************\n",
      "Folder : 2 Epoch : 2\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.227057, valid_loss: 0.205165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.801532, valid_f1: 0.809408\n",
      "save folder 2 global val max f1 model score 0.809408\n",
      "**********************************\n",
      "Folder : 2 Epoch : 3\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.157928, valid_loss: 0.150366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.847200, valid_f1: 0.838664\n",
      "save folder 2 global val max f1 model score 0.838664\n",
      "**********************************\n",
      "Folder : 2 Epoch : 4\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.124572, valid_loss: 0.136481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.903798, valid_f1: 0.910303\n",
      "save folder 2 global val max f1 model score 0.910303\n",
      "**********************************\n",
      "Folder : 2 Epoch : 5\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.107928, valid_loss: 0.109417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.928909, valid_f1: 0.931624\n",
      "save folder 2 global val max f1 model score 0.931624\n",
      "**********************************\n",
      "Folder : 2 Epoch : 6\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.099793, valid_loss: 0.105697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932308, valid_f1: 0.932353\n",
      "save folder 2 global val max f1 model score 0.932353\n",
      "**********************************\n",
      "Folder : 2 Epoch : 7\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.097338, valid_loss: 0.100477\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931870, valid_f1: 0.931494\n",
      "**********************************\n",
      "Folder : 2 Epoch : 8\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092058, valid_loss: 0.099918\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935186, valid_f1: 0.934091\n",
      "save folder 2 global val max f1 model score 0.934091\n",
      "**********************************\n",
      "Folder : 2 Epoch : 9\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.097918, valid_loss: 0.094396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.929281, valid_f1: 0.934820\n",
      "save folder 2 global val max f1 model score 0.934820\n",
      "**********************************\n",
      "Folder : 2 Epoch : 10\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.101358, valid_loss: 0.104684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.924813, valid_f1: 0.927728\n",
      "**********************************\n",
      "Folder : 2 Epoch : 11\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089846, valid_loss: 0.096005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935691, valid_f1: 0.929531\n",
      "**********************************\n",
      "Folder : 2 Epoch : 12\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087444, valid_loss: 0.095528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936782, valid_f1: 0.932859\n",
      "**********************************\n",
      "Folder : 2 Epoch : 13\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093646, valid_loss: 0.091223\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932214, valid_f1: 0.935659\n",
      "save folder 2 global val max f1 model score 0.935659\n",
      "**********************************\n",
      "Folder : 2 Epoch : 14\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093573, valid_loss: 0.092832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931129, valid_f1: 0.932476\n",
      "**********************************\n",
      "Folder : 2 Epoch : 15\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092203, valid_loss: 0.092911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932790, valid_f1: 0.933183\n",
      "**********************************\n",
      "Folder : 2 Epoch : 16\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.102310, valid_loss: 0.101745\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.924912, valid_f1: 0.930094\n",
      "**********************************\n",
      "Folder : 2 Epoch : 17\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088124, valid_loss: 0.096493\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933658, valid_f1: 0.928303\n",
      "**********************************\n",
      "Folder : 2 Epoch : 18\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088034, valid_loss: 0.089083\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935269, valid_f1: 0.936194\n",
      "save folder 2 global val max f1 model score 0.936194\n",
      "**********************************\n",
      "Folder : 2 Epoch : 19\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084896, valid_loss: 0.089700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937775, valid_f1: 0.934852\n",
      "**********************************\n",
      "Folder : 2 Epoch : 20\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084796, valid_loss: 0.087959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937564, valid_f1: 0.936900\n",
      "save folder 2 global val max f1 model score 0.936900\n",
      "**********************************\n",
      "Folder : 2 Epoch : 21\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.091493, valid_loss: 0.088534\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931412, valid_f1: 0.936342\n",
      "**********************************\n",
      "Folder : 2 Epoch : 22\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085808, valid_loss: 0.100558\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936276, valid_f1: 0.932392\n",
      "**********************************\n",
      "Folder : 2 Epoch : 23\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085966, valid_loss: 0.087537\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936078, valid_f1: 0.936894\n",
      "**********************************\n",
      "Folder : 2 Epoch : 24\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089190, valid_loss: 0.091924\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932857, valid_f1: 0.931160\n",
      "**********************************\n",
      "Folder : 2 Epoch : 25\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084965, valid_loss: 0.095691\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937302, valid_f1: 0.933617\n",
      "**********************************\n",
      "Folder : 2 Epoch : 26\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086783, valid_loss: 0.087256\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935511, valid_f1: 0.936973\n",
      "save folder 2 global val max f1 model score 0.936973\n",
      "**********************************\n",
      "Folder : 2 Epoch : 27\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084257, valid_loss: 0.088386\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937260, valid_f1: 0.936377\n",
      "**********************************\n",
      "Folder : 2 Epoch : 28\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083178, valid_loss: 0.087911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937974, valid_f1: 0.936387\n",
      "**********************************\n",
      "Folder : 2 Epoch : 29\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.71it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084788, valid_loss: 0.086414\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935449, valid_f1: 0.937022\n",
      "save folder 2 global val max f1 model score 0.937022\n",
      "**********************************\n",
      "Folder : 2 Epoch : 30\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084386, valid_loss: 0.092313\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936680, valid_f1: 0.933949\n",
      "**********************************\n",
      "Folder : 2 Epoch : 31\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084055, valid_loss: 0.090471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936831, valid_f1: 0.935428\n",
      "**********************************\n",
      "Folder : 2 Epoch : 32\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083661, valid_loss: 0.087524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936984, valid_f1: 0.934491\n",
      "**********************************\n",
      "Folder : 2 Epoch : 33\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083535, valid_loss: 0.086242\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937013, valid_f1: 0.937200\n",
      "save folder 2 global val max f1 model score 0.937200\n",
      "**********************************\n",
      "Folder : 2 Epoch : 34\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083680, valid_loss: 0.097719\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936609, valid_f1: 0.926455\n",
      "**********************************\n",
      "Folder : 2 Epoch : 35\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.091337, valid_loss: 0.088061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.930105, valid_f1: 0.936231\n",
      "**********************************\n",
      "Folder : 2 Epoch : 36\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085529, valid_loss: 0.087753\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936310, valid_f1: 0.936826\n",
      "**********************************\n",
      "Folder : 2 Epoch : 37\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087406, valid_loss: 0.086924\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935078, valid_f1: 0.936916\n",
      "**********************************\n",
      "Folder : 2 Epoch : 38\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086925, valid_loss: 0.088651\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933561, valid_f1: 0.936450\n",
      "**********************************\n",
      "Folder : 2 Epoch : 39\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083078, valid_loss: 0.087318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937777, valid_f1: 0.935941\n",
      "**********************************\n",
      "Folder : 2 Epoch : 40\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082244, valid_loss: 0.085253\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938524, valid_f1: 0.937753\n",
      "save folder 2 global val max f1 model score 0.937753\n",
      "**********************************\n",
      "Folder : 2 Epoch : 41\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081926, valid_loss: 0.086094\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938734, valid_f1: 0.937404\n",
      "**********************************\n",
      "Folder : 2 Epoch : 42\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081739, valid_loss: 0.086962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938279, valid_f1: 0.933955\n",
      "**********************************\n",
      "Folder : 2 Epoch : 43\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084250, valid_loss: 0.085809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935698, valid_f1: 0.937387\n",
      "**********************************\n",
      "Folder : 2 Epoch : 44\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082473, valid_loss: 0.086612\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938144, valid_f1: 0.936417\n",
      "**********************************\n",
      "Folder : 2 Epoch : 45\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081517, valid_loss: 0.086893\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938278, valid_f1: 0.936789\n",
      "**********************************\n",
      "Folder : 2 Epoch : 46\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083159, valid_loss: 0.086533\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937024, valid_f1: 0.935780\n",
      "**********************************\n",
      "Folder : 2 Epoch : 47\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081561, valid_loss: 0.086853\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938191, valid_f1: 0.935810\n",
      "**********************************\n",
      "Folder : 2 Epoch : 48\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083826, valid_loss: 0.085592\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936364, valid_f1: 0.937595\n",
      "**********************************\n",
      "Folder : 2 Epoch : 49\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082553, valid_loss: 0.085606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938195, valid_f1: 0.937535\n",
      "**********************************\n",
      "Folder : 2 Epoch : 50\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080355, valid_loss: 0.085082\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938985, valid_f1: 0.937475\n",
      "**********************************\n",
      "Folder : 2 Epoch : 51\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080983, valid_loss: 0.086199\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938971, valid_f1: 0.936643\n",
      "**********************************\n",
      "Folder : 2 Epoch : 52\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080390, valid_loss: 0.085001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938992, valid_f1: 0.937860\n",
      "save folder 2 global val max f1 model score 0.937860\n",
      "**********************************\n",
      "Folder : 2 Epoch : 53\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081139, valid_loss: 0.088053\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938625, valid_f1: 0.935146\n",
      "**********************************\n",
      "Folder : 2 Epoch : 54\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089392, valid_loss: 0.085099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.930648, valid_f1: 0.937721\n",
      "**********************************\n",
      "Folder : 2 Epoch : 55\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089301, valid_loss: 0.095121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932392, valid_f1: 0.931425\n",
      "**********************************\n",
      "Folder : 2 Epoch : 56\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089992, valid_loss: 0.089968\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.930131, valid_f1: 0.934102\n",
      "**********************************\n",
      "Folder : 2 Epoch : 57\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083627, valid_loss: 0.086684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936127, valid_f1: 0.936688\n",
      "**********************************\n",
      "Folder : 2 Epoch : 58\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080923, valid_loss: 0.088393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938879, valid_f1: 0.936281\n",
      "**********************************\n",
      "Folder : 2 Epoch : 59\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.77it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080442, valid_loss: 0.085218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939143, valid_f1: 0.937387\n",
      "**********************************\n",
      "Folder : 2 Epoch : 60\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080794, valid_loss: 0.085210\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938438, valid_f1: 0.937170\n",
      "**********************************\n",
      "Folder : 2 Epoch : 61\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081699, valid_loss: 0.087093\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938345, valid_f1: 0.935644\n",
      "**********************************\n",
      "Folder : 2 Epoch : 62\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083787, valid_loss: 0.087115\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936221, valid_f1: 0.936225\n",
      "**********************************\n",
      "Folder : 2 Epoch : 63\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080876, valid_loss: 0.085827\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938645, valid_f1: 0.937283\n",
      "**********************************\n",
      "Folder : 2 Epoch : 64\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079587, valid_loss: 0.084869\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939288, valid_f1: 0.937715\n",
      "**********************************\n",
      "Folder : 2 Epoch : 65\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080374, valid_loss: 0.085339\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938905, valid_f1: 0.937228\n",
      "**********************************\n",
      "Folder : 2 Epoch : 66\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079800, valid_loss: 0.086588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939658, valid_f1: 0.935280\n",
      "**********************************\n",
      "Folder : 2 Epoch : 67\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081828, valid_loss: 0.085431\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937219, valid_f1: 0.937342\n",
      "**********************************\n",
      "Folder : 2 Epoch : 68\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080236, valid_loss: 0.086116\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938944, valid_f1: 0.937295\n",
      "**********************************\n",
      "Folder : 2 Epoch : 69\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080552, valid_loss: 0.086499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938901, valid_f1: 0.936092\n",
      "**********************************\n",
      "Folder : 2 Epoch : 70\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086225, valid_loss: 0.085662\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935046, valid_f1: 0.937398\n",
      "**********************************\n",
      "Folder : 2 Epoch : 71\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081733, valid_loss: 0.088767\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938068, valid_f1: 0.935574\n",
      "**********************************\n",
      "Folder : 2 Epoch : 72\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079899, valid_loss: 0.087123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939316, valid_f1: 0.936091\n",
      "**********************************\n",
      "Folder : 2 Epoch : 73\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081680, valid_loss: 0.086192\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938271, valid_f1: 0.937187\n",
      "**********************************\n",
      "Folder : 2 Epoch : 74\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079702, valid_loss: 0.086219\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939319, valid_f1: 0.937230\n",
      "**********************************\n",
      "Folder : 2 Epoch : 75\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082287, valid_loss: 0.085963\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937827, valid_f1: 0.937171\n",
      "**********************************\n",
      "Folder : 2 Epoch : 76\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079651, valid_loss: 0.092042\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938931, valid_f1: 0.932267\n",
      "**********************************\n",
      "Folder : 2 Epoch : 77\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080316, valid_loss: 0.086925\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938933, valid_f1: 0.934719\n",
      "**********************************\n",
      "Folder : 2 Epoch : 78\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079219, valid_loss: 0.085859\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939587, valid_f1: 0.936939\n",
      "**********************************\n",
      "Folder : 2 Epoch : 79\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080904, valid_loss: 0.084806\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/32 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937678, valid_f1: 0.938350\n",
      "save folder 2 global val max f1 model score 0.938350\n",
      "Folder 2 finally best global max f1 score is 0.9383498959280793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:07<00:00,  4.27it/s]\n",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold : 3\n",
      "**********************************\n",
      "Folder : 3 Epoch : 0\n",
      "Curr learning_rate: 0.001000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 1.030879, valid_loss: 0.734162\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.400620, valid_f1: 0.514196\n",
      "save folder 3 global val max f1 model score 0.514196\n",
      "**********************************\n",
      "Folder : 3 Epoch : 1\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.414693, valid_loss: 0.440687\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.700508, valid_f1: 0.695806\n",
      "save folder 3 global val max f1 model score 0.695806\n",
      "**********************************\n",
      "Folder : 3 Epoch : 2\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.250074, valid_loss: 0.229080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.794517, valid_f1: 0.809748\n",
      "save folder 3 global val max f1 model score 0.809748\n",
      "**********************************\n",
      "Folder : 3 Epoch : 3\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.76it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.159375, valid_loss: 0.152183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.842905, valid_f1: 0.842401\n",
      "save folder 3 global val max f1 model score 0.842401\n",
      "**********************************\n",
      "Folder : 3 Epoch : 4\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.128362, valid_loss: 0.129107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.886542, valid_f1: 0.880239\n",
      "save folder 3 global val max f1 model score 0.880239\n",
      "**********************************\n",
      "Folder : 3 Epoch : 5\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.108858, valid_loss: 0.138102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.926556, valid_f1: 0.917208\n",
      "save folder 3 global val max f1 model score 0.917208\n",
      "**********************************\n",
      "Folder : 3 Epoch : 6\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:35<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.111579, valid_loss: 0.106550\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.920845, valid_f1: 0.934605\n",
      "save folder 3 global val max f1 model score 0.934605\n",
      "**********************************\n",
      "Folder : 3 Epoch : 7\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.096428, valid_loss: 0.105109\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933775, valid_f1: 0.932569\n",
      "**********************************\n",
      "Folder : 3 Epoch : 8\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.094228, valid_loss: 0.098934\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934094, valid_f1: 0.935594\n",
      "save folder 3 global val max f1 model score 0.935594\n",
      "**********************************\n",
      "Folder : 3 Epoch : 9\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.75it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089833, valid_loss: 0.097563\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936231, valid_f1: 0.936054\n",
      "save folder 3 global val max f1 model score 0.936054\n",
      "**********************************\n",
      "Folder : 3 Epoch : 10\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.091575, valid_loss: 0.094724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933558, valid_f1: 0.937206\n",
      "save folder 3 global val max f1 model score 0.937206\n",
      "**********************************\n",
      "Folder : 3 Epoch : 11\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.100057, valid_loss: 0.097908\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.926955, valid_f1: 0.934086\n",
      "**********************************\n",
      "Folder : 3 Epoch : 12\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088531, valid_loss: 0.094173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935603, valid_f1: 0.937080\n",
      "**********************************\n",
      "Folder : 3 Epoch : 13\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086265, valid_loss: 0.092611\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936438, valid_f1: 0.936458\n",
      "**********************************\n",
      "Folder : 3 Epoch : 14\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089324, valid_loss: 0.095697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933020, valid_f1: 0.932872\n",
      "**********************************\n",
      "Folder : 3 Epoch : 15\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087538, valid_loss: 0.093225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934125, valid_f1: 0.935792\n",
      "**********************************\n",
      "Folder : 3 Epoch : 16\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085545, valid_loss: 0.094211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936183, valid_f1: 0.933658\n",
      "**********************************\n",
      "Folder : 3 Epoch : 17\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088664, valid_loss: 0.094780\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933126, valid_f1: 0.931640\n",
      "**********************************\n",
      "Folder : 3 Epoch : 18\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.427261, valid_loss: 0.118672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.816263, valid_f1: 0.932411\n",
      "**********************************\n",
      "Folder : 3 Epoch : 19\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087219, valid_loss: 0.277812\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935572, valid_f1: 0.908170\n",
      "**********************************\n",
      "Folder : 3 Epoch : 20\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.138215, valid_loss: 0.121285\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.928895, valid_f1: 0.931014\n",
      "**********************************\n",
      "Folder : 3 Epoch : 21\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.71it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092655, valid_loss: 0.103521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.928913, valid_f1: 0.934362\n",
      "**********************************\n",
      "Folder : 3 Epoch : 22\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093155, valid_loss: 0.097344\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935432, valid_f1: 0.935874\n",
      "**********************************\n",
      "Folder : 3 Epoch : 23\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.74it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083707, valid_loss: 0.097036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937853, valid_f1: 0.935911\n",
      "**********************************\n",
      "Folder : 3 Epoch : 24\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087982, valid_loss: 0.096019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936577, valid_f1: 0.936189\n",
      "**********************************\n",
      "Folder : 3 Epoch : 25\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.73it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083459, valid_loss: 0.094007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937348, valid_f1: 0.936773\n",
      "**********************************\n",
      "Folder : 3 Epoch : 26\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086741, valid_loss: 0.092985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937085, valid_f1: 0.936250\n",
      "**********************************\n",
      "Folder : 3 Epoch : 27\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082926, valid_loss: 0.092616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937910, valid_f1: 0.937260\n",
      "save folder 3 global val max f1 model score 0.937260\n",
      "**********************************\n",
      "Folder : 3 Epoch : 28\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086001, valid_loss: 0.091161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937292, valid_f1: 0.937272\n",
      "save folder 3 global val max f1 model score 0.937272\n",
      "**********************************\n",
      "Folder : 3 Epoch : 29\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082643, valid_loss: 0.091927\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937972, valid_f1: 0.937222\n",
      "**********************************\n",
      "Folder : 3 Epoch : 30\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084747, valid_loss: 0.090733\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937189, valid_f1: 0.936759\n",
      "**********************************\n",
      "Folder : 3 Epoch : 31\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.72it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083399, valid_loss: 0.091188\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937192, valid_f1: 0.936307\n",
      "**********************************\n",
      "Folder : 3 Epoch : 32\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084132, valid_loss: 0.089672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937091, valid_f1: 0.937611\n",
      "save folder 3 global val max f1 model score 0.937611\n",
      "**********************************\n",
      "Folder : 3 Epoch : 33\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082247, valid_loss: 0.091547\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937809, valid_f1: 0.935148\n",
      "**********************************\n",
      "Folder : 3 Epoch : 34\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084610, valid_loss: 0.092992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936482, valid_f1: 0.926842\n",
      "**********************************\n",
      "Folder : 3 Epoch : 35\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083325, valid_loss: 0.089997\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936638, valid_f1: 0.937524\n",
      "**********************************\n",
      "Folder : 3 Epoch : 36\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082650, valid_loss: 0.090287\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938004, valid_f1: 0.937094\n",
      "**********************************\n",
      "Folder : 3 Epoch : 37\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081952, valid_loss: 0.091820\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938098, valid_f1: 0.933084\n",
      "**********************************\n",
      "Folder : 3 Epoch : 38\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083138, valid_loss: 0.089460\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937722, valid_f1: 0.937570\n",
      "**********************************\n",
      "Folder : 3 Epoch : 39\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081890, valid_loss: 0.089765\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938313, valid_f1: 0.937063\n",
      "**********************************\n",
      "Folder : 3 Epoch : 40\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085055, valid_loss: 0.088767\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935200, valid_f1: 0.937100\n",
      "**********************************\n",
      "Folder : 3 Epoch : 41\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082566, valid_loss: 0.091334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937720, valid_f1: 0.936560\n",
      "**********************************\n",
      "Folder : 3 Epoch : 42\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088100, valid_loss: 0.089513\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.929764, valid_f1: 0.937355\n",
      "**********************************\n",
      "Folder : 3 Epoch : 43\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092862, valid_loss: 0.111239\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.925142, valid_f1: 0.908637\n",
      "**********************************\n",
      "Folder : 3 Epoch : 44\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085320, valid_loss: 0.092687\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934713, valid_f1: 0.930157\n",
      "**********************************\n",
      "Folder : 3 Epoch : 45\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083224, valid_loss: 0.089637\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935965, valid_f1: 0.936865\n",
      "**********************************\n",
      "Folder : 3 Epoch : 46\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082514, valid_loss: 0.090161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937673, valid_f1: 0.936521\n",
      "**********************************\n",
      "Folder : 3 Epoch : 47\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082856, valid_loss: 0.090714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937752, valid_f1: 0.936631\n",
      "**********************************\n",
      "Folder : 3 Epoch : 48\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082271, valid_loss: 0.089021\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937222, valid_f1: 0.937675\n",
      "save folder 3 global val max f1 model score 0.937675\n",
      "**********************************\n",
      "Folder : 3 Epoch : 49\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081263, valid_loss: 0.089032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938178, valid_f1: 0.937719\n",
      "save folder 3 global val max f1 model score 0.937719\n",
      "**********************************\n",
      "Folder : 3 Epoch : 50\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081873, valid_loss: 0.088405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937894, valid_f1: 0.938121\n",
      "save folder 3 global val max f1 model score 0.938121\n",
      "**********************************\n",
      "Folder : 3 Epoch : 51\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081033, valid_loss: 0.097299\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938547, valid_f1: 0.925955\n",
      "**********************************\n",
      "Folder : 3 Epoch : 52\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.091853, valid_loss: 0.088593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.928748, valid_f1: 0.935725\n",
      "**********************************\n",
      "Folder : 3 Epoch : 53\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087557, valid_loss: 0.089475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932477, valid_f1: 0.937360\n",
      "**********************************\n",
      "Folder : 3 Epoch : 54\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082723, valid_loss: 0.094210\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936375, valid_f1: 0.933154\n",
      "**********************************\n",
      "Folder : 3 Epoch : 55\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081395, valid_loss: 0.088510\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937974, valid_f1: 0.937180\n",
      "**********************************\n",
      "Folder : 3 Epoch : 56\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081531, valid_loss: 0.088551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938464, valid_f1: 0.937614\n",
      "**********************************\n",
      "Folder : 3 Epoch : 57\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081187, valid_loss: 0.089694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938687, valid_f1: 0.937109\n",
      "**********************************\n",
      "Folder : 3 Epoch : 58\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081129, valid_loss: 0.088099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938435, valid_f1: 0.937328\n",
      "**********************************\n",
      "Folder : 3 Epoch : 59\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081497, valid_loss: 0.087963\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938248, valid_f1: 0.937618\n",
      "**********************************\n",
      "Folder : 3 Epoch : 60\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081492, valid_loss: 0.089218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937919, valid_f1: 0.937025\n",
      "**********************************\n",
      "Folder : 3 Epoch : 61\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082030, valid_loss: 0.087880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937645, valid_f1: 0.938246\n",
      "save folder 3 global val max f1 model score 0.938246\n",
      "**********************************\n",
      "Folder : 3 Epoch : 62\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081923, valid_loss: 0.088047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937195, valid_f1: 0.937635\n",
      "**********************************\n",
      "Folder : 3 Epoch : 63\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084192, valid_loss: 0.103270\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936377, valid_f1: 0.911137\n",
      "**********************************\n",
      "Folder : 3 Epoch : 64\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.092908, valid_loss: 0.090467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.929205, valid_f1: 0.936066\n",
      "**********************************\n",
      "Folder : 3 Epoch : 65\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084412, valid_loss: 0.088158\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934655, valid_f1: 0.937524\n",
      "**********************************\n",
      "Folder : 3 Epoch : 66\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081372, valid_loss: 0.090682\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937184, valid_f1: 0.937013\n",
      "**********************************\n",
      "Folder : 3 Epoch : 67\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080524, valid_loss: 0.089380\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938801, valid_f1: 0.936040\n",
      "**********************************\n",
      "Folder : 3 Epoch : 68\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081043, valid_loss: 0.088345\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938193, valid_f1: 0.937689\n",
      "**********************************\n",
      "Folder : 3 Epoch : 69\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079859, valid_loss: 0.087708\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939114, valid_f1: 0.938046\n",
      "**********************************\n",
      "Folder : 3 Epoch : 70\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080700, valid_loss: 0.087599\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938471, valid_f1: 0.938270\n",
      "save folder 3 global val max f1 model score 0.938270\n",
      "**********************************\n",
      "Folder : 3 Epoch : 71\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082169, valid_loss: 0.087704\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937130, valid_f1: 0.937617\n",
      "**********************************\n",
      "Folder : 3 Epoch : 72\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080101, valid_loss: 0.089287\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938527, valid_f1: 0.937755\n",
      "**********************************\n",
      "Folder : 3 Epoch : 73\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080666, valid_loss: 0.088169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938526, valid_f1: 0.937018\n",
      "**********************************\n",
      "Folder : 3 Epoch : 74\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080726, valid_loss: 0.090236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938398, valid_f1: 0.936575\n",
      "**********************************\n",
      "Folder : 3 Epoch : 75\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080408, valid_loss: 0.087976\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938297, valid_f1: 0.938238\n",
      "**********************************\n",
      "Folder : 3 Epoch : 76\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079809, valid_loss: 0.088133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939307, valid_f1: 0.937673\n",
      "**********************************\n",
      "Folder : 3 Epoch : 77\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081491, valid_loss: 0.087756\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937393, valid_f1: 0.937951\n",
      "**********************************\n",
      "Folder : 3 Epoch : 78\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.112206, valid_loss: 0.091069\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.909616, valid_f1: 0.936757\n",
      "**********************************\n",
      "Folder : 3 Epoch : 79\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079676, valid_loss: 0.094916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/32 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939235, valid_f1: 0.931587\n",
      "Folder 3 finally best global max f1 score is 0.9382696762713475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:07<00:00,  4.03it/s]\n",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold : 4\n",
      "**********************************\n",
      "Folder : 4 Epoch : 0\n",
      "Curr learning_rate: 0.001000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.977344, valid_loss: 0.679580\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.426660, valid_f1: 0.537710\n",
      "save folder 4 global val max f1 model score 0.537710\n",
      "**********************************\n",
      "Folder : 4 Epoch : 1\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.375053, valid_loss: 0.394507\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.727463, valid_f1: 0.748152\n",
      "save folder 4 global val max f1 model score 0.748152\n",
      "**********************************\n",
      "Folder : 4 Epoch : 2\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.226186, valid_loss: 0.193162\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.808858, valid_f1: 0.828806\n",
      "save folder 4 global val max f1 model score 0.828806\n",
      "**********************************\n",
      "Folder : 4 Epoch : 3\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.148558, valid_loss: 0.147314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.840811, valid_f1: 0.839133\n",
      "save folder 4 global val max f1 model score 0.839133\n",
      "**********************************\n",
      "Folder : 4 Epoch : 4\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.117706, valid_loss: 0.127312\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.893460, valid_f1: 0.851884\n",
      "save folder 4 global val max f1 model score 0.851884\n",
      "**********************************\n",
      "Folder : 4 Epoch : 5\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.132797, valid_loss: 0.113992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.900769, valid_f1: 0.920160\n",
      "save folder 4 global val max f1 model score 0.920160\n",
      "**********************************\n",
      "Folder : 4 Epoch : 6\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.109024, valid_loss: 0.121948\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.924687, valid_f1: 0.928963\n",
      "save folder 4 global val max f1 model score 0.928963\n",
      "**********************************\n",
      "Folder : 4 Epoch : 7\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.099603, valid_loss: 0.101203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931321, valid_f1: 0.935590\n",
      "save folder 4 global val max f1 model score 0.935590\n",
      "**********************************\n",
      "Folder : 4 Epoch : 8\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.097880, valid_loss: 0.097296\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.927546, valid_f1: 0.936334\n",
      "save folder 4 global val max f1 model score 0.936334\n",
      "**********************************\n",
      "Folder : 4 Epoch : 9\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.095556, valid_loss: 0.107255\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932327, valid_f1: 0.930016\n",
      "**********************************\n",
      "Folder : 4 Epoch : 10\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.098730, valid_loss: 0.094449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.927969, valid_f1: 0.936838\n",
      "save folder 4 global val max f1 model score 0.936838\n",
      "**********************************\n",
      "Folder : 4 Epoch : 11\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.091195, valid_loss: 0.098310\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934418, valid_f1: 0.933371\n",
      "**********************************\n",
      "Folder : 4 Epoch : 12\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088164, valid_loss: 0.093885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935994, valid_f1: 0.935322\n",
      "**********************************\n",
      "Folder : 4 Epoch : 13\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086824, valid_loss: 0.092687\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935883, valid_f1: 0.936790\n",
      "**********************************\n",
      "Folder : 4 Epoch : 14\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088369, valid_loss: 0.091839\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934657, valid_f1: 0.936698\n",
      "**********************************\n",
      "Folder : 4 Epoch : 15\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085976, valid_loss: 0.095612\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936688, valid_f1: 0.931461\n",
      "**********************************\n",
      "Folder : 4 Epoch : 16\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088943, valid_loss: 0.097251\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.932914, valid_f1: 0.932389\n",
      "**********************************\n",
      "Folder : 4 Epoch : 17\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.087782, valid_loss: 0.091236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934105, valid_f1: 0.936900\n",
      "save folder 4 global val max f1 model score 0.936900\n",
      "**********************************\n",
      "Folder : 4 Epoch : 18\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.088515, valid_loss: 0.092613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934023, valid_f1: 0.935004\n",
      "**********************************\n",
      "Folder : 4 Epoch : 19\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085191, valid_loss: 0.089899\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937135, valid_f1: 0.937006\n",
      "save folder 4 global val max f1 model score 0.937006\n",
      "**********************************\n",
      "Folder : 4 Epoch : 20\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083460, valid_loss: 0.090710\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937624, valid_f1: 0.936399\n",
      "**********************************\n",
      "Folder : 4 Epoch : 21\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084275, valid_loss: 0.090691\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936991, valid_f1: 0.935570\n",
      "**********************************\n",
      "Folder : 4 Epoch : 22\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.71it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085136, valid_loss: 0.090027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936375, valid_f1: 0.936767\n",
      "**********************************\n",
      "Folder : 4 Epoch : 23\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083184, valid_loss: 0.089302\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937631, valid_f1: 0.937279\n",
      "save folder 4 global val max f1 model score 0.937279\n",
      "**********************************\n",
      "Folder : 4 Epoch : 24\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085802, valid_loss: 0.090678\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936247, valid_f1: 0.935684\n",
      "**********************************\n",
      "Folder : 4 Epoch : 25\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.093666, valid_loss: 0.089692\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.925664, valid_f1: 0.936735\n",
      "**********************************\n",
      "Folder : 4 Epoch : 26\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.086400, valid_loss: 0.129809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935571, valid_f1: 0.904099\n",
      "**********************************\n",
      "Folder : 4 Epoch : 27\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.71it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.090672, valid_loss: 0.089688\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931722, valid_f1: 0.937480\n",
      "save folder 4 global val max f1 model score 0.937480\n",
      "**********************************\n",
      "Folder : 4 Epoch : 28\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082615, valid_loss: 0.090659\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938059, valid_f1: 0.936103\n",
      "**********************************\n",
      "Folder : 4 Epoch : 29\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083422, valid_loss: 0.088725\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937185, valid_f1: 0.937620\n",
      "save folder 4 global val max f1 model score 0.937620\n",
      "**********************************\n",
      "Folder : 4 Epoch : 30\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082009, valid_loss: 0.089149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938094, valid_f1: 0.937339\n",
      "**********************************\n",
      "Folder : 4 Epoch : 31\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084629, valid_loss: 0.088408\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935659, valid_f1: 0.937290\n",
      "**********************************\n",
      "Folder : 4 Epoch : 32\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082547, valid_loss: 0.089859\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938070, valid_f1: 0.936217\n",
      "**********************************\n",
      "Folder : 4 Epoch : 33\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082156, valid_loss: 0.089448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937878, valid_f1: 0.936993\n",
      "**********************************\n",
      "Folder : 4 Epoch : 34\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084537, valid_loss: 0.091032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935672, valid_f1: 0.934474\n",
      "**********************************\n",
      "Folder : 4 Epoch : 35\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084456, valid_loss: 0.091629\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936580, valid_f1: 0.935679\n",
      "**********************************\n",
      "Folder : 4 Epoch : 36\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084784, valid_loss: 0.087635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935858, valid_f1: 0.938002\n",
      "save folder 4 global val max f1 model score 0.938002\n",
      "**********************************\n",
      "Folder : 4 Epoch : 37\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081172, valid_loss: 0.090945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938690, valid_f1: 0.934084\n",
      "**********************************\n",
      "Folder : 4 Epoch : 38\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.089524, valid_loss: 0.088996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.931078, valid_f1: 0.937383\n",
      "**********************************\n",
      "Folder : 4 Epoch : 39\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082161, valid_loss: 0.096109\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937682, valid_f1: 0.932691\n",
      "**********************************\n",
      "Folder : 4 Epoch : 40\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085978, valid_loss: 0.089584\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.933024, valid_f1: 0.936206\n",
      "**********************************\n",
      "Folder : 4 Epoch : 41\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082015, valid_loss: 0.092524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937715, valid_f1: 0.934630\n",
      "**********************************\n",
      "Folder : 4 Epoch : 42\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081183, valid_loss: 0.087542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938227, valid_f1: 0.937807\n",
      "**********************************\n",
      "Folder : 4 Epoch : 43\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.65it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080593, valid_loss: 0.088099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938736, valid_f1: 0.938036\n",
      "save folder 4 global val max f1 model score 0.938036\n",
      "**********************************\n",
      "Folder : 4 Epoch : 44\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081595, valid_loss: 0.087181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938404, valid_f1: 0.938256\n",
      "save folder 4 global val max f1 model score 0.938256\n",
      "**********************************\n",
      "Folder : 4 Epoch : 45\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081761, valid_loss: 0.088448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938024, valid_f1: 0.937707\n",
      "**********************************\n",
      "Folder : 4 Epoch : 46\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084022, valid_loss: 0.090437\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935733, valid_f1: 0.935471\n",
      "**********************************\n",
      "Folder : 4 Epoch : 47\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081304, valid_loss: 0.087449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938275, valid_f1: 0.937984\n",
      "**********************************\n",
      "Folder : 4 Epoch : 48\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085018, valid_loss: 0.093502\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935611, valid_f1: 0.931447\n",
      "**********************************\n",
      "Folder : 4 Epoch : 49\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.65it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.083346, valid_loss: 0.091471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.936823, valid_f1: 0.934497\n",
      "**********************************\n",
      "Folder : 4 Epoch : 50\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.65it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.085421, valid_loss: 0.090975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.934380, valid_f1: 0.934190\n",
      "**********************************\n",
      "Folder : 4 Epoch : 51\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081729, valid_loss: 0.089378\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938066, valid_f1: 0.937803\n",
      "**********************************\n",
      "Folder : 4 Epoch : 52\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080041, valid_loss: 0.087099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938759, valid_f1: 0.938316\n",
      "save folder 4 global val max f1 model score 0.938316\n",
      "**********************************\n",
      "Folder : 4 Epoch : 53\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079793, valid_loss: 0.087729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938970, valid_f1: 0.938037\n",
      "**********************************\n",
      "Folder : 4 Epoch : 54\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080310, valid_loss: 0.087587\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938780, valid_f1: 0.938195\n",
      "**********************************\n",
      "Folder : 4 Epoch : 55\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081451, valid_loss: 0.087171\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937325, valid_f1: 0.938219\n",
      "**********************************\n",
      "Folder : 4 Epoch : 56\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.71it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081177, valid_loss: 0.094610\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938538, valid_f1: 0.935276\n",
      "**********************************\n",
      "Folder : 4 Epoch : 57\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084432, valid_loss: 0.087368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935811, valid_f1: 0.938147\n",
      "**********************************\n",
      "Folder : 4 Epoch : 58\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080837, valid_loss: 0.090439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938633, valid_f1: 0.935935\n",
      "**********************************\n",
      "Folder : 4 Epoch : 59\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.084088, valid_loss: 0.086776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935264, valid_f1: 0.938161\n",
      "**********************************\n",
      "Folder : 4 Epoch : 60\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:36<00:00,  1.71it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080791, valid_loss: 0.087565\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937814, valid_f1: 0.937750\n",
      "**********************************\n",
      "Folder : 4 Epoch : 61\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.69it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080152, valid_loss: 0.089058\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938979, valid_f1: 0.935598\n",
      "**********************************\n",
      "Folder : 4 Epoch : 62\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.70it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.080387, valid_loss: 0.087488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938649, valid_f1: 0.937645\n",
      "**********************************\n",
      "Folder : 4 Epoch : 63\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082371, valid_loss: 0.086753\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935870, valid_f1: 0.938334\n",
      "save folder 4 global val max f1 model score 0.938334\n",
      "**********************************\n",
      "Folder : 4 Epoch : 64\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.62it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.082476, valid_loss: 0.087488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.935194, valid_f1: 0.937950\n",
      "**********************************\n",
      "Folder : 4 Epoch : 65\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079323, valid_loss: 0.094445\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938906, valid_f1: 0.933984\n",
      "**********************************\n",
      "Folder : 4 Epoch : 66\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079757, valid_loss: 0.086602\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938741, valid_f1: 0.938407\n",
      "save folder 4 global val max f1 model score 0.938407\n",
      "**********************************\n",
      "Folder : 4 Epoch : 67\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079268, valid_loss: 0.087601\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939185, valid_f1: 0.935655\n",
      "**********************************\n",
      "Folder : 4 Epoch : 68\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081227, valid_loss: 0.087425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937098, valid_f1: 0.937706\n",
      "**********************************\n",
      "Folder : 4 Epoch : 69\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.64it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079935, valid_loss: 0.092244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938068, valid_f1: 0.934526\n",
      "**********************************\n",
      "Folder : 4 Epoch : 70\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081514, valid_loss: 0.086822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937588, valid_f1: 0.938338\n",
      "**********************************\n",
      "Folder : 4 Epoch : 71\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079398, valid_loss: 0.100979\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939076, valid_f1: 0.929167\n",
      "**********************************\n",
      "Folder : 4 Epoch : 72\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.081524, valid_loss: 0.086826\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.937467, valid_f1: 0.938375\n",
      "**********************************\n",
      "Folder : 4 Epoch : 73\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079227, valid_loss: 0.088575\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938936, valid_f1: 0.937386\n",
      "**********************************\n",
      "Folder : 4 Epoch : 74\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079228, valid_loss: 0.087304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939012, valid_f1: 0.937782\n",
      "**********************************\n",
      "Folder : 4 Epoch : 75\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.68it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:03<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.078902, valid_loss: 0.086907\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939127, valid_f1: 0.938182\n",
      "**********************************\n",
      "Folder : 4 Epoch : 76\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.65it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.078804, valid_loss: 0.086939\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939765, valid_f1: 0.938363\n",
      "**********************************\n",
      "Folder : 4 Epoch : 77\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.66it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.078494, valid_loss: 0.087177\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939402, valid_f1: 0.938354\n",
      "**********************************\n",
      "Folder : 4 Epoch : 78\n",
      "Curr learning_rate: 0.002000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:38<00:00,  1.64it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079053, valid_loss: 0.089987\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/63 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.939015, valid_f1: 0.935478\n",
      "**********************************\n",
      "Folder : 4 Epoch : 79\n",
      "Curr learning_rate: 0.000400000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 63/63 [00:37<00:00,  1.67it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:04<00:00,  3.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_loss: 0.079804, valid_loss: 0.087705\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/32 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_f1: 0.938050, valid_f1: 0.937883\n",
      "Folder 4 finally best global max f1 score is 0.9384072723611167\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [00:08<00:00,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4h 9min 29s, sys: 30min 1s, total: 4h 39min 30s\n",
      "Wall time: 4h 41min 49s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test_y = np.zeros([int(2000000/GROUP_BATCH_SIZE), GROUP_BATCH_SIZE, 1])\n",
    "test_dataset = IronDataset(test, test_y, flip=False)\n",
    "test_dataloader = DataLoader(test_dataset, NNBATCHSIZE, shuffle=False, num_workers=8, pin_memory=True)\n",
    "test_preds_all = np.zeros((2000000, 11))\n",
    "\n",
    "\n",
    "oof_score = []\n",
    "for index, (train_index, val_index, _) in enumerate(new_splits[0:], start=0):\n",
    "    print(\"Fold : {}\".format(index))\n",
    "    train_dataset = IronDataset(train[train_index], train_tr[train_index], seq_len=GROUP_BATCH_SIZE, flip=flip, noise_level=noise)\n",
    "    train_dataloader = DataLoader(train_dataset, NNBATCHSIZE, shuffle=True, num_workers=8, pin_memory=True)\n",
    "\n",
    "    valid_dataset = IronDataset(train[val_index], train_tr[val_index], seq_len=GROUP_BATCH_SIZE, flip=False)\n",
    "    valid_dataloader = DataLoader(valid_dataset, NNBATCHSIZE, shuffle=False, num_workers=4, pin_memory=True)\n",
    "\n",
    "    it = 0\n",
    "    model = Classifier()\n",
    "    model = model.cuda()\n",
    "\n",
    "    early_stopping = EarlyStopping(patience=40, is_maximize=True,\n",
    "                                   checkpoint_path=os.path.join(outdir, \"gru_clean_checkpoint_fold_{}_iter_{}.pt\".format(index,\n",
    "                                                                                                             it)))\n",
    "\n",
    "    weight = None#cal_weights()\n",
    "    criterion = nn.CrossEntropyLoss(weight=weight)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n",
    "    \n",
    "    optimizer = torchcontrib.optim.SWA(optimizer, swa_start=10, swa_freq=5, swa_lr=0.002)\n",
    "\n",
    "\n",
    "    schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3, factor=0.2)\n",
    "    avg_train_losses, avg_valid_losses = [], []\n",
    "\n",
    "    \n",
    "\n",
    "    for epoch in range(EPOCHS):\n",
    "        print('**********************************')\n",
    "        print(\"Folder : {} Epoch : {}\".format(index, epoch))\n",
    "        print(\"Curr learning_rate: {:0.9f}\".format(optimizer.param_groups[0]['lr']))\n",
    "        train_losses, valid_losses = [], []\n",
    "        tr_loss_cls_item, val_loss_cls_item = [], []\n",
    "\n",
    "        model.train()  # prep model for training\n",
    "        train_preds, train_true = torch.Tensor([]).cuda(), torch.LongTensor([]).cuda()#.to(device)\n",
    "\n",
    "        for x, y in tqdm(train_dataloader):\n",
    "            x = x.cuda()\n",
    "            y = y.cuda()\n",
    "            #print(x.shape)\n",
    "\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            #loss_fn(model(input), target).backward()\n",
    "            \n",
    "        \n",
    "        \n",
    "            #optimizer.zero_grad()\n",
    "            predictions = model(x)\n",
    "\n",
    "            predictions_ = predictions.view(-1, predictions.shape[-1])\n",
    "            y_ = y.view(-1)\n",
    "\n",
    "            loss = criterion(predictions_, y_)\n",
    "\n",
    "            # backward pass: compute gradient of the loss with respect to model parameters\n",
    "            loss.backward()\n",
    "            # perform a single optimization step (parameter update)\n",
    "            optimizer.step()\n",
    "            \n",
    "            #schedular.step()\n",
    "            # record training lossa\n",
    "            train_losses.append(loss.item())\n",
    "            train_true = torch.cat([train_true, y_], 0)\n",
    "            train_preds = torch.cat([train_preds, predictions_], 0)\n",
    "\n",
    "        #model.eval()  # prep model for evaluation\n",
    "        optimizer.swap_swa_sgd()\n",
    "        val_preds, val_true = torch.Tensor([]).cuda(), torch.LongTensor([]).cuda()\n",
    "        print('EVALUATION')\n",
    "        with torch.no_grad():\n",
    "            for x, y in tqdm(valid_dataloader):\n",
    "                x = x.cuda()#.to(device)\n",
    "                y = y.cuda()#..to(device)\n",
    "\n",
    "                predictions = model(x)\n",
    "                predictions_ = predictions.view(-1, predictions.shape[-1])\n",
    "                y_ = y.view(-1)\n",
    "\n",
    "                loss = criterion(predictions_, y_)\n",
    "\n",
    "                valid_losses.append(loss.item())\n",
    "\n",
    "\n",
    "                val_true = torch.cat([val_true, y_], 0)\n",
    "                val_preds = torch.cat([val_preds, predictions_], 0)\n",
    "\n",
    "        # calculate average loss over an epoch\n",
    "        train_loss = np.average(train_losses)\n",
    "        valid_loss = np.average(valid_losses)\n",
    "        avg_train_losses.append(train_loss)\n",
    "        avg_valid_losses.append(valid_loss)\n",
    "        print(\"train_loss: {:0.6f}, valid_loss: {:0.6f}\".format(train_loss, valid_loss))\n",
    "\n",
    "        train_score = f1_score(train_true.cpu().detach().numpy(), train_preds.cpu().detach().numpy().argmax(1),\n",
    "                               labels=list(range(11)), average='macro')\n",
    "\n",
    "        val_score = f1_score(val_true.cpu().detach().numpy(), val_preds.cpu().detach().numpy().argmax(1),\n",
    "                             labels=list(range(11)), average='macro')\n",
    "\n",
    "        schedular.step(val_score)\n",
    "        print(\"train_f1: {:0.6f}, valid_f1: {:0.6f}\".format(train_score, val_score))\n",
    "        res = early_stopping(val_score, model)\n",
    "        #print('fres:', res)\n",
    "        if  res == 2:\n",
    "            print(\"Early Stopping\")\n",
    "            print('folder %d global best val max f1 model score %f' % (index, early_stopping.best_score))\n",
    "            break\n",
    "        elif res == 1:\n",
    "            print('save folder %d global val max f1 model score %f' % (index, val_score))\n",
    "    print('Folder {} finally best global max f1 score is {}'.format(index, early_stopping.best_score))\n",
    "    oof_score.append(round(early_stopping.best_score, 6))\n",
    "    \n",
    "    model.eval()\n",
    "    pred_list = []\n",
    "    with torch.no_grad():\n",
    "        for x, y in tqdm(test_dataloader):\n",
    "            \n",
    "            x = x.cuda()\n",
    "            y = y.cuda()\n",
    "\n",
    "            predictions = model(x)\n",
    "            predictions_ = predictions.view(-1, predictions.shape[-1]) # shape [128, 4000, 11]\n",
    "            #print(predictions.shape, F.softmax(predictions_, dim=1).cpu().numpy().shape)\n",
    "            pred_list.append(F.softmax(predictions_, dim=1).cpu().numpy()) # shape (512000, 11)\n",
    "            #a = input()\n",
    "        test_preds = np.vstack(pred_list) # shape [2000000, 11]\n",
    "        test_preds_all += test_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "id": "cPy8MbF1ZDOE",
    "outputId": "bfbca118-8be0-43cc-c564-fc997282fb53"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all folder score is:[0.938465, 0.939936, 0.93835, 0.93827, 0.938407]\n",
      "OOF mean score is: 0.938686\n",
      "Generate submission.............\n",
      "over\n"
     ]
    }
   ],
   "source": [
    "print('all folder score is:%s'%str(oof_score))\n",
    "print('OOF mean score is: %f'% (sum(oof_score)/len(oof_score)))\n",
    "print('Generate submission.............')\n",
    "submission_csv_path = '/content/gdrive/My Drive/data/sample_submission.csv'\n",
    "ss = pd.read_csv(submission_csv_path, dtype={'time': str})\n",
    "test_preds_all = test_preds_all / np.sum(test_preds_all, axis=1)[:, None]\n",
    "test_pred_frame = pd.DataFrame({'time': ss['time'].astype(str),\n",
    "                                'open_channels': np.argmax(test_preds_all, axis=1)})\n",
    "test_pred_frame.to_csv(\"/content/gdrive/My Drive/data/gru_submission.csv\", index=False)\n",
    "print('over')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f1Grcw7Kml41"
   },
   "source": [
    "**Результат:** 0.941 на public lb"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Wavenet.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}