Image-side/Finetuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Finetuning.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "270ac4482b364dc4998ac46e4a5e14c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_09c1f958fe23464bbbe13370b39d22e3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8a259fea99034f19b08b6cde1d10df8b",
              "IPY_MODEL_22a765dee94f4c758141c6ff3de8e90a"
            ]
          }
        },
        "09c1f958fe23464bbbe13370b39d22e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a259fea99034f19b08b6cde1d10df8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2e0a3484e4c24a65a25271677c8fde35",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 102502400,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 102502400,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cebdc17ae0de4f5089f981d3de029e66"
          }
        },
        "22a765dee94f4c758141c6ff3de8e90a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e38f568a7c7d47ab83fa6bdb8ddfcf67",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 97.8M/97.8M [04:45&lt;00:00, 359kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b12a15396d944291a624e251ab8fbb19"
          }
        },
        "2e0a3484e4c24a65a25271677c8fde35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cebdc17ae0de4f5089f981d3de029e66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e38f568a7c7d47ab83fa6bdb8ddfcf67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b12a15396d944291a624e251ab8fbb19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "VYpv-elsrlED",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://food-x.s3.amazonaws.com/train.tar"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KLyantI0rmLX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!tar -xvf train.tar"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8imNI-DrqgW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d1e44863-0f66-44ab-bbe3-7278f83d4e65"
      },
      "source": [
        "%cd train_set"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/train_set\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qb6j9nbHrsof",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from PIL import Image\n",
        "import glob\n",
        "images = []\n",
        "for filename in glob.glob('/content/train_set/*.jpg'): \n",
        "    im=Image.open(filename)\n",
        "    images.append(im)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5XAkxLbAryqw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.models as models\n",
        "import torchvision.transforms as transforms\n",
        "from torch.autograd import Variable\n",
        "from PIL import Image"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3XBojhIfUNTl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess(img):\n",
        "  scaler = transforms.Resize((224,224))\n",
        "  normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                                   std=[0.229, 0.224, 0.225])\n",
        "  to_tensor = transforms.ToTensor()\n",
        "  t_img = Variable(normalize(to_tensor(scaler(img))).unsqueeze(0))\n",
        "  return t_img"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "32Fz8TpiNELH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preprocess_image = []\n",
        "for img in images:\n",
        "  t_img = preprocess(img)\n",
        "  preprocess_image.append(t_img)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XmPP07E6MtyE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "image_tensor = torch.Tensor(len(preprocess_image), 3, 224, 224)\n",
        "torch.cat(preprocess_image, out=image_tensor)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KoTmyn0KYWzx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "270ac4482b364dc4998ac46e4a5e14c1",
            "09c1f958fe23464bbbe13370b39d22e3",
            "8a259fea99034f19b08b6cde1d10df8b",
            "22a765dee94f4c758141c6ff3de8e90a",
            "2e0a3484e4c24a65a25271677c8fde35",
            "cebdc17ae0de4f5089f981d3de029e66",
            "e38f568a7c7d47ab83fa6bdb8ddfcf67",
            "b12a15396d944291a624e251ab8fbb19"
          ]
        },
        "outputId": "6b7e74d3-9075-4925-c832-db42c409694a"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.models as models\n",
        "from torch.autograd import Variable\n",
        "resnet50 = models.resnet50(pretrained=True)\n",
        "modules=list(resnet50.children())[:-1]\n",
        "resnet50=nn.Sequential(*modules)\n",
        "#for p in resnet50.parameters():\n",
        "   # p.requires_grad = True"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet50-19c8e357.pth\" to /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "270ac4482b364dc4998ac46e4a5e14c1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=102502400.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hCmGsz2y3iv9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0aea1cca-7c42-465f-b132-a3aa34f9a51e"
      },
      "source": [
        "resnet50(image_tensor).size()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 2048, 1, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7lpIL7id3wmE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "27c46820-59fe-4bc5-c57a-c1f519b2377d"
      },
      "source": [
        "image_tensor.size()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([10, 3, 224, 224])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OGF0jp5x2rtU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "training_data = []\n",
        "for image in image_tensor:\n",
        "  training_data.append((torch.rand(2048),image))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3_phPbuS6HlZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "recipes = torch.rand(10,2048)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "be9f0JH14XGq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.nn.utils.clip_grad import clip_grad_norm_\n",
        "from torch.optim import Adam, SGD\n",
        "lr = 0.1\n",
        "num_epochs = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3LUfGIj_1jjR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = resnet50\n",
        "loss_function = nn.CosineEmbeddingLoss()\n",
        "params = list(model.parameters())\n",
        "optimizer = Adam(params, lr=lr, weight_decay=5e-4)\n",
        "for epoch in range(num_epochs):  \n",
        "    print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
        "    for recipe, image in training_data[:1]:\n",
        "        model.zero_grad()\n",
        "        image = image.unsqueeze(0)\n",
        "        output = model(image)\n",
        "        output = torch.squeeze(output)\n",
        "        loss = loss_function(output.unsqueeze(0),recipe.unsqueeze(0), torch.tensor([1]))\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "    print(loss)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2jU89JcJ72fO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4db2fedd-c7cb-4b68-a434-95bceabacd28"
      },
      "source": [
        "y = torch.empty(10).bernoulli_().mul_(2).sub_(1)\n",
        "y"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 1., -1., -1.,  1., -1., -1.,  1.,  1.,  1., -1.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JeBIiFPh7Bid",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        },
        "outputId": "dfc424c7-dfaf-40a3-de45-bcf33f5d6c21"
      },
      "source": [
        "model = resnet50\n",
        "loss_function = nn.CosineEmbeddingLoss()\n",
        "params = list(model.parameters())\n",
        "optimizer = Adam(params, lr=lr, weight_decay=5e-4)\n",
        "for epoch in range(num_epochs):  \n",
        "    print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
        "    #for recipe, image in training_data[:1]:\n",
        "    model.zero_grad()\n",
        "    #image = image.unsqueeze(0)\n",
        "    output = model(image_tensor)\n",
        "    output = torch.squeeze(output)\n",
        "    loss = loss_function(output,recipes, y)\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    print(loss)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0/19\n",
            "tensor(0.4075, grad_fn=<MeanBackward0>)\n",
            "Epoch 1/19\n",
            "tensor(0.4106, grad_fn=<MeanBackward0>)\n",
            "Epoch 2/19\n",
            "tensor(0.4116, grad_fn=<MeanBackward0>)\n",
            "Epoch 3/19\n",
            "tensor(0.4026, grad_fn=<MeanBackward0>)\n",
            "Epoch 4/19\n",
            "tensor(0.3876, grad_fn=<MeanBackward0>)\n",
            "Epoch 5/19\n",
            "tensor(0.3679, grad_fn=<MeanBackward0>)\n",
            "Epoch 6/19\n",
            "tensor(0.3376, grad_fn=<MeanBackward0>)\n",
            "Epoch 7/19\n",
            "tensor(0.3864, grad_fn=<MeanBackward0>)\n",
            "Epoch 8/19\n",
            "tensor(0.4330, grad_fn=<MeanBackward0>)\n",
            "Epoch 9/19\n",
            "tensor(0.4001, grad_fn=<MeanBackward0>)\n",
            "Epoch 10/19\n",
            "tensor(0.4621, grad_fn=<MeanBackward0>)\n",
            "Epoch 11/19\n",
            "tensor(0.3849, grad_fn=<MeanBackward0>)\n",
            "Epoch 12/19\n",
            "tensor(0.3893, grad_fn=<MeanBackward0>)\n",
            "Epoch 13/19\n",
            "tensor(0.3665, grad_fn=<MeanBackward0>)\n",
            "Epoch 14/19\n",
            "tensor(0.4013, grad_fn=<MeanBackward0>)\n",
            "Epoch 15/19\n",
            "tensor(0.4296, grad_fn=<MeanBackward0>)\n",
            "Epoch 16/19\n",
            "tensor(0.4139, grad_fn=<MeanBackward0>)\n",
            "Epoch 17/19\n",
            "tensor(0.3925, grad_fn=<MeanBackward0>)\n",
            "Epoch 18/19\n",
            "tensor(0.3761, grad_fn=<MeanBackward0>)\n",
            "Epoch 19/19\n",
            "tensor(0.3648, grad_fn=<MeanBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}