chapter 6/Tensorflow VR Part 1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow Visual Recognition Sample Application Part 1\n",
    "## Define the model metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "pixiedust": {
     "displayParams": {}
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import requests\n",
    "models = {\n",
    "    \"mobilenet\": {\n",
    "        \"base_url\":\"https://github.com/DTAIEB/Thoughtful-Data-Science/raw/master/chapter%206/Visual%20Recognition/mobilenet_v1_0.50_224\",\n",
    "        \"model_file_url\": \"frozen_graph.pb\",\n",
    "        \"label_file\": \"labels.txt\",\n",
    "        \"output_layer\": \"MobilenetV1/Predictions/Softmax\"\n",
    "    }\n",
    "}\n",
    "\n",
    "# helper method for reading attributes from the model metadata\n",
    "def get_model_attribute(model, key, default_value = None):\n",
    "    if key not in model:\n",
    "        if default_value is None:\n",
    "            raise Exception(\"Require model attribute {} not found\".format(key))\n",
    "        return default_value\n",
    "    return model[key]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper methods for loading the graph and labels for a given model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper method for resolving url relative to the selected model\n",
    "def get_url(model, path):\n",
    "    return model[\"base_url\"] + \"/\" + path\n",
    "    \n",
    "# Download the serialized model and create a TensorFlow graph\n",
    "def load_graph(model):\n",
    "    graph = tf.Graph()\n",
    "    graph_def = tf.GraphDef()\n",
    "    graph_def.ParseFromString(\n",
    "        requests.get( get_url( model, model[\"model_file_url\"] ) ).content\n",
    "    )\n",
    "    with graph.as_default():\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph\n",
    "\n",
    "# Load the labels\n",
    "def load_labels(model, as_json = False):\n",
    "    labels = [line.rstrip() \\\n",
    "        for line in requests.get( get_url( model, model[\"label_file\"] ) ).text.split(\"\\n\") \\\n",
    "        if line != \"\"]\n",
    "    if as_json:\n",
    "        return [{\"index\": item.split(\":\")[0], \"label\" : item.split(\":\")[1]} for item in labels]\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use BeautifulSoup to scrape the images from a given url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as BS\n",
    "import re\n",
    "\n",
    "# return an array of all the images scraped from an html page\n",
    "def get_image_urls(url):\n",
    "    # Instantiate a BeautifulSoup parser\n",
    "    soup = BS(requests.get(url).text, \"html.parser\")\n",
    "    \n",
    "    # Local helper method for extracting url\n",
    "    def extract_url(val):\n",
    "        m = re.match(r\"url\\((.*)\\)\", val)\n",
    "        val = m.group(1) if m is not None else val\n",
    "        return \"http:\" + val if val.startswith(\"//\") else val\n",
    "    \n",
    "    # List comprehension that look for <img> elements and backgroud-image styles\n",
    "    return [extract_url(imgtag['src']) for imgtag in soup.find_all('img')] + [ \\\n",
    "        extract_url(val.strip()) for key,val in \\\n",
    "        [tuple(selector.split(\":\")) for elt in soup.select(\"[style]\") \\\n",
    "            for selector in elt[\"style\"].strip(\" ;\").split(\";\")] \\\n",
    "            if key.strip().lower()=='background-image' \\\n",
    "        ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper method for downloading an image into a temp file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tempfile\n",
    "def download_image(url):\n",
    "    response = requests.get(url, stream=True)\n",
    "    if response.status_code == 200:\n",
    "        with tempfile.NamedTemporaryFile(delete=False) as f:\n",
    "            for chunk in response.iter_content(2048):\n",
    "                f.write(chunk)\n",
    "            return f.name\n",
    "    else:\n",
    "        raise Exception(\"Unable to download image: {}\".format(response.status_code))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decode an image into a tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decode a given image into a tensor\n",
    "def read_tensor_from_image_file(model, file_name):\n",
    "    file_reader = tf.read_file(file_name, \"file_reader\")\n",
    "    if file_name.endswith(\".png\"):\n",
    "        image_reader = tf.image.decode_png(file_reader, channels = 3,name='png_reader')\n",
    "    elif file_name.endswith(\".gif\"):\n",
    "        image_reader = tf.squeeze(tf.image.decode_gif(file_reader,name='gif_reader'))\n",
    "    elif file_name.endswith(\".bmp\"):\n",
    "        image_reader = tf.image.decode_bmp(file_reader, name='bmp_reader')\n",
    "    else:\n",
    "        image_reader = tf.image.decode_jpeg(file_reader, channels = 3, name='jpeg_reader')\n",
    "    float_caster = tf.cast(image_reader, tf.float32)\n",
    "    dims_expander = tf.expand_dims(float_caster, 0);\n",
    "    \n",
    "    # Read some info from the model metadata, providing default values\n",
    "    input_height = get_model_attribute(model, \"input_height\", 224)\n",
    "    input_width = get_model_attribute(model, \"input_width\", 224)\n",
    "    input_mean = get_model_attribute(model, \"input_mean\", 0)\n",
    "    input_std = get_model_attribute(model, \"input_std\", 255)\n",
    "        \n",
    "    resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])\n",
    "    normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])\n",
    "    sess = tf.Session()\n",
    "    result = sess.run(normalized)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Score_image method that run the model and return the top 5 candidate answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# classify an image given its url\n",
    "def score_image(graph, model, url):\n",
    "    # Get the input and output layer from the model\n",
    "    input_layer = get_model_attribute(model, \"input_layer\", \"input\")\n",
    "    output_layer = get_model_attribute(model, \"output_layer\")\n",
    "    \n",
    "    # Download the image and build a tensor from its data\n",
    "    t = read_tensor_from_image_file(model, download_image(url))\n",
    "    \n",
    "    # Retrieve the tensors corresponding to the input and output layers\n",
    "    input_tensor = graph.get_tensor_by_name(\"import/\" + input_layer + \":0\");\n",
    "    output_tensor = graph.get_tensor_by_name(\"import/\" + output_layer + \":0\");\n",
    "\n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        # Execute the output, overriding the input tensor with the one corresponding\n",
    "        # to the image in the feed_dict argument\n",
    "        results = sess.run(output_tensor, {input_tensor: t})\n",
    "    results = np.squeeze(results)\n",
    "    # select the top 5 candidate and match them to the labels\n",
    "    top_k = results.argsort()[-5:][::-1]\n",
    "    labels = load_labels(model)\n",
    "    return [(labels[i].split(\":\")[1], results[i]) for i in top_k]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the model using a Flickr page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for https://geo.yahoo.com/b?s=792600534: \n",
      "\t[('nail', 0.034935154), ('screw', 0.03144558), ('puck, hockey puck', 0.03032596), ('envelope', 0.0285034), ('Band Aid', 0.027891463)]\n",
      "Results for http://c1.staticflickr.com/6/5598/14934282524_344c84246b_n.jpg: \n",
      "\t[('Egyptian cat', 0.4644194), ('tiger cat', 0.1485573), ('tabby, tabby cat', 0.09759513), ('plastic bag', 0.03814263), ('Siamese cat, Siamese', 0.033892646)]\n",
      "Results for http://c1.staticflickr.com/4/3677/13545844805_170ec3746b_n.jpg: \n",
      "\t[('tabby, tabby cat', 0.7330132), ('Egyptian cat', 0.14256532), ('tiger cat', 0.11719289), ('plastic bag', 0.0028653105), ('bow tie, bow-tie, bowtie', 0.00082955)]\n",
      "Results for http://c1.staticflickr.com/6/5170/5372754294_db6acaa1e5_n.jpg: \n",
      "\t[('Persian cat', 0.607673), ('Angora, Angora rabbit', 0.20204937), ('hamster', 0.02988311), ('Egyptian cat', 0.027227053), ('lynx, catamount', 0.018035706)]\n",
      "Results for http://c1.staticflickr.com/6/5589/14818641818_b0058c0cfc_m.jpg: \n",
      "\t[('Egyptian cat', 0.5786173), ('tabby, tabby cat', 0.27942237), ('tiger cat', 0.11966114), ('lynx, catamount', 0.016066141), ('plastic bag', 0.002206809)]\n",
      "Results for http://c1.staticflickr.com/6/5036/5881933297_7974eaff82_n.jpg: \n",
      "\t[('tiger cat', 0.26617262), ('tabby, tabby cat', 0.2417825), ('Persian cat', 0.18471399), ('lynx, catamount', 0.11543496), ('Egyptian cat', 0.025188642)]\n",
      "Results for http://c1.staticflickr.com/3/2602/3977203168_b9d02a0233.jpg: \n",
      "\t[('tabby, tabby cat', 0.75482476), ('tiger cat', 0.13780454), ('Egyptian cat', 0.05675489), ('Siamese cat, Siamese', 0.02073992), ('lynx, catamount', 0.010187127)]\n",
      "Results for http://c1.staticflickr.com/8/7401/16393044637_72e93d96b6_n.jpg: \n",
      "\t[('Egyptian cat', 0.67294717), ('tiger cat', 0.18149199), ('tabby, tabby cat', 0.0952419), ('lynx, catamount', 0.025225954), ('candle, taper, wax light', 0.003860443)]\n",
      "Results for http://c1.staticflickr.com/9/8110/8594699278_dd256c10fd_m.jpg: \n",
      "\t[('tabby, tabby cat', 0.5829553), ('Egyptian cat', 0.15930973), ('tiger cat', 0.12964381), ('lynx, catamount', 0.11114485), ('plastic bag', 0.006467772)]\n",
      "Results for http://c1.staticflickr.com/8/7023/6581178955_7e23af8bf9_m.jpg: \n",
      "\t[('tabby, tabby cat', 0.28574014), ('Egyptian cat', 0.190615), ('plastic bag', 0.17165014), ('lynx, catamount', 0.101593874), ('tiger cat', 0.040527806)]\n",
      "Results for http://c1.staticflickr.com/8/7313/9775005856_9b5e0ebe16_n.jpg: \n",
      "\t[('tiger cat', 0.40977326), ('tabby, tabby cat', 0.31697693), ('Egyptian cat', 0.16972947), ('lynx, catamount', 0.059500016), ('washer, automatic washer, washing machine', 0.0046033794)]\n",
      "Results for http://c1.staticflickr.com/8/7496/16236770082_205f4e358f_n.jpg: \n",
      "\t[('Egyptian cat', 0.40310237), ('Siamese cat, Siamese', 0.23720524), ('tiger cat', 0.100198396), ('tabby, tabby cat', 0.08537914), ('plastic bag', 0.0352822)]\n",
      "Results for http://c1.staticflickr.com/8/7049/13244364473_7b71bc5a4f_n.jpg: \n",
      "\t[('Egyptian cat', 0.59387493), ('candle, taper, wax light', 0.057717346), ('paper towel', 0.046397187), ('plastic bag', 0.035106137), ('tabby, tabby cat', 0.018382242)]\n",
      "Results for http://c1.staticflickr.com/4/3753/9837176706_9ecc1cddac_n.jpg: \n",
      "\t[('tabby, tabby cat', 0.55699265), ('Egyptian cat', 0.19758604), ('tiger cat', 0.12088148), ('lynx, catamount', 0.057880934), ('plastic bag', 0.01653284)]\n",
      "Results for http://c1.staticflickr.com/4/3488/4051998735_5b4863ac11_m.jpg: \n",
      "\t[('Egyptian cat', 0.5310361), ('tabby, tabby cat', 0.26919606), ('tiger cat', 0.13531871), ('lynx, catamount', 0.050503224), ('washer, automatic washer, washing machine', 0.0053878534)]\n",
      "Results for http://c1.staticflickr.com/9/8335/8086459588_46aae939c8.jpg: \n",
      "\t[('Siamese cat, Siamese', 0.827261), ('mouse, computer mouse', 0.046974737), ('screen, CRT screen', 0.029382586), ('carton', 0.0076049017), ('lynx, catamount', 0.0067297667)]\n",
      "Results for http://c1.staticflickr.com/8/7472/16230028882_c03cd6f2cc_n.jpg: \n",
      "\t[('tiger cat', 0.5394526), ('lynx, catamount', 0.14366476), ('Egyptian cat', 0.10943988), ('red fox, Vulpes vulpes', 0.07641454), ('tabby, tabby cat', 0.034076575)]\n",
      "Results for http://c1.staticflickr.com/4/3940/15504684310_f555c88915_n.jpg: \n",
      "\t[('tabby, tabby cat', 0.49280357), ('Egyptian cat', 0.31668788), ('tiger cat', 0.12977621), ('lynx, catamount', 0.022205332), ('plastic bag', 0.008769177)]\n",
      "Results for http://c1.staticflickr.com/9/8630/16556634997_ef0f9dd5f1_n.jpg: \n",
      "\t[('West Highland white terrier', 0.8534684), ('Angora, Angora rabbit', 0.038167812), ('Samoyed, Samoyede', 0.024762549), ('Scotch terrier, Scottish terrier, Scottie', 0.01685713), ('Persian cat', 0.01484343)]\n",
      "Results for http://c1.staticflickr.com/6/5226/5674849391_824822628c_n.jpg: \n",
      "\t[('tiger cat', 0.45084468), ('tabby, tabby cat', 0.40245533), ('Egyptian cat', 0.11048719), ('lynx, catamount', 0.024745336), ('tiger, Panthera tigris', 0.0064596836)]\n",
      "Results for http://c1.staticflickr.com/3/2234/1704658865_3b982b56cf_m.jpg: \n",
      "\t[('Angora, Angora rabbit', 0.21852449), ('Egyptian cat', 0.19025268), ('tabby, tabby cat', 0.14283349), ('Persian cat', 0.085699804), ('tiger cat', 0.06147669)]\n",
      "Results for http://c1.staticflickr.com/2/1361/5110233061_aa3b1c47ef_n.jpg: \n",
      "\t[('tabby, tabby cat', 0.6095775), ('tiger cat', 0.24819912), ('Egyptian cat', 0.13453156), ('lynx, catamount', 0.0021140918), ('carton', 0.0015312452)]\n",
      "Results for http://c1.staticflickr.com/4/3294/2434900370_17c1221ccf_n.jpg: \n",
      "\t[('Egyptian cat', 0.4372107), ('tabby, tabby cat', 0.26445335), ('tiger cat', 0.13057052), ('bow tie, bow-tie, bowtie', 0.06754344), ('lynx, catamount', 0.037636597)]\n",
      "Results for http://c1.staticflickr.com/3/2858/12174748174_27491cde33_n.jpg: \n",
      "\t[('tiger cat', 0.4069278), ('tabby, tabby cat', 0.23834446), ('Egyptian cat', 0.23789576), ('lynx, catamount', 0.11284405), ('tiger, Panthera tigris', 0.0008611009)]\n",
      "Results for http://c1.staticflickr.com/4/3674/13336301695_1cab4f5c85_n.jpg: \n",
      "\t[('weasel', 0.25950897), ('black-footed ferret, ferret, Mustela nigripes', 0.1795659), ('polecat, fitch, foulmart, foumart, Mustela putorius', 0.15248777), ('mink', 0.07626065), ('Egyptian cat', 0.04768039)]\n"
     ]
    }
   ],
   "source": [
    "model = models['mobilenet']\n",
    "graph = load_graph(model)\n",
    "image_urls = get_image_urls(\"https://www.flickr.com/search/?text=cats\")\n",
    "for url in image_urls:\n",
    "    results = score_image(graph, model, url)\n",
    "    print(\"Results for {}: \\n\\t{}\".format(url, results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Edit Metadata",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}