imageCaptioning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "imageCaptioning.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "JDAgp_sTX_st",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######## Install Tensorflow version 1.13.2\n",
        "!pip install tensorflow==1.13.2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LH2Ikk20YiH_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######## Official git repo\n",
        "##\n",
        "##     https://www.github.com/purvesh-linux11/imageCaptioning.git\n",
        "##\n",
        "##### clone project moduls\n",
        "\n",
        "!git clone https://www.github.com/purvesh-linux11/imageCaptioning.git\n",
        "!cp -r /content/imageCaptioning/* /content/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2AkEa0BupDsF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##### copy model from drive and extract here\n",
        "\n",
        "!cp -r /content/drive/'My Drive'/MODELS/image_captioning_model.zip /content/  \n",
        "!unzip image_captioning_model.zip\n",
        "!cp -r /content/image_captioning_model/* /content/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sYZGzNViwPQ1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##### copy photos from drive\n",
        "\n",
        "!cp -r /content/drive/'My Drive'/photos/* /content/photos/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xDwlCAOMRG5q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#############################################################\n",
        "##\n",
        "##\n",
        "## Download Model and place it current directory \n",
        "##                  (Link in Readme file)\n",
        "##\n",
        "##############################################################"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ErYFuETaEGX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import\n",
        "from __future__ import division\n",
        "from __future__ import print_function\n",
        "\n",
        "import math\n",
        "import os\n",
        "from os import listdir, path\n",
        "\n",
        "import tensorflow as tf\n",
        "import cv2\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from IPython import display\n",
        "\n",
        "import PIL.Image\n",
        "import PIL.ImageOps\n",
        "import PIL.ImageFont\n",
        "import PIL.ImageDraw\n",
        "import textwrap\n",
        "from io import BytesIO\n",
        "\n",
        "from im2txt import configuration\n",
        "from im2txt import inference_wrapper\n",
        "from im2txt.inference_utils import caption_generator\n",
        "from im2txt.inference_utils import vocabulary"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fN-tgbJMbNHt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Choose the trained model --> current is 2\n",
        "model_number = \"2\"\n",
        "model_path = \"/content/model.ckpt-\"+model_number+\"000000\"   # Give model path\n",
        "vocab_path = \"/content/word_counts\"+model_number+\".txt\"     # Give word_counts file path\n",
        "tf.logging.set_verbosity(tf.logging.INFO)\n",
        "\n",
        "# Build the inference graph.\n",
        "g = tf.Graph()\n",
        "with g.as_default():\n",
        "    model = inference_wrapper.InferenceWrapper()\n",
        "    restore_fn = model.build_graph_from_config(configuration.ModelConfig(), model_path)\n",
        "g.finalize()\n",
        "\n",
        "# Create the vocabulary.\n",
        "vocab = vocabulary.Vocabulary(vocab_path) \n",
        "\n",
        "#######################################################\n",
        "### if tensorflow version is <1.13.2 then you have to check variables name as per tensorflow version\n",
        "\n",
        "OLD_CHECKPOINT_FILE = \"/content/model.ckpt-2000000\"\n",
        "NEW_CHECKPOINT_FILE = \"/content/model.ckpt-2000000\"\n",
        "\n",
        "import tensorflow as tf\n",
        "vars_to_rename = {\n",
        "    \"lstm/BasicLSTMCell/Linear/Matrix\": \"lstm/basic_lstm_cell/kernel\",\n",
        "    \"lstm/BasicLSTMCell/Linear/Bias\": \"lstm/basic_lstm_cell/bias\",\n",
        "}\n",
        "new_checkpoint_vars = {}\n",
        "reader = tf.train.NewCheckpointReader(OLD_CHECKPOINT_FILE)\n",
        "for old_name in reader.get_variable_to_shape_map():\n",
        "  if old_name in vars_to_rename:\n",
        "    new_name = vars_to_rename[old_name]\n",
        "  else:\n",
        "    new_name = old_name\n",
        "  new_checkpoint_vars[new_name] = tf.Variable(reader.get_tensor(old_name))\n",
        "\n",
        "init = tf.global_variables_initializer()\n",
        "saver = tf.train.Saver(new_checkpoint_vars)\n",
        "\n",
        "with tf.Session() as sess:\n",
        "  sess.run(init)\n",
        "  saver.save(sess, NEW_CHECKPOINT_FILE)\n",
        "\n",
        "#######################################################\n",
        "\n",
        "sess = tf.Session(graph=g)\n",
        "# Load the model from checkpoint.\n",
        "restore_fn(sess)\n",
        "\n",
        "# Prepare the caption generator. Here we are implicitly using the default\n",
        "# beam search parameters. See caption_generator.py for a description of the\n",
        "# available beam search parameters.\n",
        "generator = caption_generator.CaptionGenerator(model, vocab)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DIZLzNZIbgU-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "image_path = \"./photos/\"    #### provide path where image is stored\n",
        "filename = listdir(image_path)\n",
        "filenames = [f for f in filename if '.jpg' in f or '.png' in f or '.jpeg' in f]\n",
        "store = open('/content/results/logs.txt','w')   #### directory to store captions file\n",
        "\n",
        "for file in filenames:\n",
        "    try:\n",
        "        img = PIL.Image.open(image_path+file).convert('RGBA')\n",
        "        box = PIL.Image.new('RGBA', img.size, (255,255,255,0))\n",
        "        draw = PIL.ImageDraw.Draw(box)\n",
        "        image = open(image_path+file,'rb').read() # Read the image as bytes\n",
        "        captions = generator.beam_search(sess, image)\n",
        "        for i, caption in enumerate(captions):\n",
        "            # Ignore begin and end words.\n",
        "            sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]\n",
        "            sentence = \" \".join(sentence)\n",
        "            if i==0:\n",
        "              print(file+\" : %s \" % (sentence))\n",
        "              #create log file image_name--image_caption\n",
        "              store.write(file+\" : \"+sentence+\"\\n\")\n",
        "    except KeyboardInterrupt:\n",
        "        store.close()\n",
        "        break\n",
        "store.close()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}