save_best_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "374d38b7",
   "metadata": {},
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e1f0694f-82ff-4dfc-9eb3-8a0b3e331ebf",
   "metadata": {},
   "source": [
    "## Save Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3e1c4602",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f3c9f5fd-e585-47c3-9419-0fa3d31b8ff3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import os.path as osp\n",
    "if osp.exists(\"imagecaption\"):\n",
    "    os.chdir(\"imagecaption\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec4460c2-3fee-4e34-8997-d5c43e7ae7f3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install -r train/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ff8bd00d-cb33-41c2-bc7b-51d9d9743f13",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os.path as osp\n",
    "import sys\n",
    "train_dir = osp.abspath(\"train\")\n",
    "if train_dir not in sys.path:\n",
    "    sys.path.insert(0, train_dir)\n",
    "    \n",
    "import pytorch_lightning as pl\n",
    "from huggingface_hub import HfApi, notebook_login\n",
    "\n",
    "\n",
    "pl.seed_everything(42, workers=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15b57429",
   "metadata": {},
   "source": [
    "### Login to HuggingFace"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "123a3fc1",
   "metadata": {},
   "source": [
    "### Run PL Trainer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c1689e83",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "args = {\n",
    "  \"best_model\": \"soul11zz/image-caption-description-large-v2\",\n",
    "  \"checkpoint\": \"z:/checkpoint.ckpt\",\n",
    "  \"dataset\": \"soul11zz/image-description-large\",\n",
    "  \"metric\": \"semantic\",\n",
    "}\n",
    "\n",
    "args = Namespace(**args)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "598c756f-8817-4a1b-9cda-db254ed69592",
   "metadata": {},
   "source": [
    "### Upload Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9ac42dfa-9325-41c2-ac07-21bf507a3aa2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-01-27T02:43:24.497998Z",
     "iopub.status.busy": "2023-01-27T02:43:24.497370Z",
     "iopub.status.idle": "2023-01-27T02:43:42.016340Z",
     "shell.execute_reply": "2023-01-27T02:43:42.015693Z",
     "shell.execute_reply.started": "2023-01-27T02:43:24.497977Z"
    }
   },
   "outputs": [],
   "source": [
    "from train import save_best_model\n",
    "\n",
    "from transformers import GitForCausalLM, GitProcessor\n",
    "from pl_module import ImageCaptioningModule\n",
    "model = GitForCausalLM.from_pretrained(\"soul11zz/image-caption-description-large\")\n",
    "processor = GitProcessor.from_pretrained(\"soul11zz/image-caption-description-large\")\n",
    "best_model = ImageCaptioningModule.load_from_checkpoint(args.checkpoint, processor=processor, model=model)\n",
    "\n",
    "model_repo = args.best_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a0fff9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ImageCaptioningModule(\n",
       "  (model): GitForCausalLM(\n",
       "    (git): GitModel(\n",
       "      (embeddings): GitEmbeddings(\n",
       "        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "        (position_embeddings): Embedding(1024, 768)\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (image_encoder): GitVisionModel(\n",
       "        (vision_model): GitVisionTransformer(\n",
       "          (embeddings): GitVisionEmbeddings(\n",
       "            (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
       "            (position_embedding): Embedding(257, 1024)\n",
       "          )\n",
       "          (pre_layrnorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder): GitVisionEncoder(\n",
       "            (layers): ModuleList(\n",
       "              (0): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (1): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (2): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (3): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (4): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (5): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (6): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (7): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (8): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (9): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (10): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (11): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (12): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (13): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (14): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (15): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (16): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (17): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (18): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (19): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (20): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (21): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (22): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "              (23): GitVisionEncoderLayer(\n",
       "                (self_attn): GitVisionAttention(\n",
       "                  (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                  (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "                (mlp): GitVisionMLP(\n",
       "                  (activation_fn): QuickGELUActivation()\n",
       "                  (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                  (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                )\n",
       "                (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (post_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "      (encoder): GitEncoder(\n",
       "        (layer): ModuleList(\n",
       "          (0): GitLayer(\n",
       "            (attention): GitAttention(\n",
       "              (self): GitSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): GitSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): GitIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): GitOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (1): GitLayer(\n",
       "            (attention): GitAttention(\n",
       "              (self): GitSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): GitSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): GitIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): GitOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (2): GitLayer(\n",
       "            (attention): GitAttention(\n",
       "              (self): GitSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): GitSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): GitIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): GitOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (3): GitLayer(\n",
       "            (attention): GitAttention(\n",
       "              (self): GitSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): GitSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): GitIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): GitOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (4): GitLayer(\n",
       "            (attention): GitAttention(\n",
       "              (self): GitSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): GitSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): GitIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): GitOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (5): GitLayer(\n",
       "            (attention): GitAttention(\n",
       "              (self): GitSelfAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (output): GitSelfOutput(\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "            (intermediate): GitIntermediate(\n",
       "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (intermediate_act_fn): GELUActivation()\n",
       "            )\n",
       "            (output): GitOutput(\n",
       "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (visual_projection): GitProjection(\n",
       "        (visual_projection): Sequential(\n",
       "          (0): Linear(in_features=1024, out_features=768, bias=True)\n",
       "          (1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (output): Linear(in_features=768, out_features=30522, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "save_best_model(best_model, args.best_model, save_dir=\"tmp/best_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ae97a1d",
   "metadata": {},
   "source": [
    "### Evaluate Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "56482e98",
   "metadata": {},
   "outputs": [],
   "source": [
    "from eval import score_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6ac256ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "222f0778f7fa41ca90aa69b3b20e6859",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/2.92k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\berno\\anaconda3\\envs\\soul\\lib\\site-packages\\huggingface_hub\\file_download.py:127: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\berno\\.cache\\huggingface\\hub. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "989683057420480282dd500cd143074c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.58G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "492e1ee5f37143408ec75c773064ff9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/503 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c46fa2fd9b040ecbf8e795a774d1ef3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/476 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c95d7b0aac67419aa2d22f41c4e7ffca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d1533f584f94b01b16d56882477c5e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/711k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9bd376da0a8461594cf0fd5c796ddd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "2023-02-25-12-28-40  WARNING    Using custom data configuration soul11zz--image-description-large-c50a0c78e57a1f07\n",
      "2023-02-25-12-28-41  WARNING    Found cached dataset parquet (C:/Users/berno/.cache/huggingface/datasets/soul11zz___parquet/soul11zz--image-description-large-c50a0c78e57a1f07/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "2023-02-25-12-28-46  WARNING    Using custom data configuration soul11zz--image-description-large-c50a0c78e57a1f07\n",
      "2023-02-25-12-28-46  WARNING    Found cached dataset parquet (C:/Users/berno/.cache/huggingface/datasets/soul11zz___parquet/soul11zz--image-description-large-c50a0c78e57a1f07/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "c:\\Users\\berno\\anaconda3\\envs\\soul\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:236: PossibleUserWarning: The dataloader, test_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7207127b577c4b0f9e82a53b45a7761e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-25-12-28-51  INFO       Load pretrained SentenceTransformer: sentence-transformers/all-mpnet-base-v2\n",
      "2023-02-25-12-28-52  INFO       Use pytorch device: cuda\n",
      "2023-02-25-14-02-45  INFO       Average SEMANTIC: 0.7266618486408225\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">        Test metric        </span>┃<span style=\"font-weight: bold\">       DataLoader 0        </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">         semantic          </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.7266618609428406     </span>│\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m        semantic         \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.7266618609428406    \u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────────┴───────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "score_model(args.best_model, args.dataset, args.metric)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "78bfdcee45ab360ff98997b0fdb262d833775071286b5e4a7c2a2c9a5871b9ea"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}