2. Benchmark Keras pretrained models on ImageNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.3.0'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys, os, time, re, gc\n",
    "from pathlib import Path\n",
    "from glob import glob\n",
    "\n",
    "# Select GPU\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.applications import vgg16, vgg19, resnet_v2\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make sure GPU is available\n",
    "tf.config.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Determine paths and directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_imagenet_val_dataset = Path(\"data/\") # path/to/data/\n",
    "dir_images = Path(\"data/val\") # path/to/images/directory\n",
    "path_labels = Path(\"data/ILSVRC2012_validation_ground_truth.txt\")\n",
    "path_synset_words = Path(\"data/synset_words.txt\")\n",
    "path_meta = Path(\"data/meta.mat\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_val_paths = glob(str(path_imagenet_val_dataset / \"x_val*.npy\"))\n",
    "\n",
    "# Sort filenames in ascending order\n",
    "x_val_paths.sort(key=lambda f: int(re.sub('\\D', '', f)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val = np.load(str(path_imagenet_val_dataset / \"y_val.npy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val_one_hot = to_categorical(y_val, 1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_k_accuracy(y_true, y_pred, k=1, tf_enabled=True):\n",
    "    '''\n",
    "    Calculates top_k accuracy of predictions. Expects both y_true and y_pred to be one-hot encoded.\n",
    "    numpy implementation is from: https://github.com/chainer/chainer/issues/606\n",
    "    '''\n",
    "\n",
    "    if tf_enabled:\n",
    "        argsorted_y = tf.argsort(y_pred)[:,-k:]\n",
    "        matches = tf.cast(tf.math.reduce_any(tf.transpose(argsorted_y) == tf.argmax(y_true, axis=1, output_type=tf.int32), axis=0), tf.float32)\n",
    "        return tf.math.reduce_mean(matches).numpy()\n",
    "    else:\n",
    "        argsorted_y = np.argsort(y_pred)[:,-k:]\n",
    "        return np.any(argsorted_y.T == y_true.argmax(axis=1), axis=0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "K.clear_session()\n",
    "model = vgg19.VGG19()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  5.0% completed.\n",
      " 10.0% completed.\n",
      " 15.0% completed.\n",
      " 20.0% completed.\n",
      " 25.0% completed.\n",
      " 30.0% completed.\n",
      " 35.0% completed.\n",
      " 40.0% completed.\n",
      " 45.0% completed.\n",
      " 50.0% completed.\n",
      " 55.0% completed.\n",
      " 60.0% completed.\n",
      " 65.0% completed.\n",
      " 70.0% completed.\n",
      " 75.0% completed.\n",
      " 80.0% completed.\n",
      " 85.0% completed.\n",
      " 90.0% completed.\n",
      " 95.0% completed.\n",
      "100.0% completed.\n"
     ]
    }
   ],
   "source": [
    "y_pred = None\n",
    "for i, x_val_path in enumerate(x_val_paths):\n",
    "    \n",
    "    x_val = np.load(x_val_path).astype('float32') # loaded as RGB\n",
    "    x_val = vgg19.preprocess_input(x_val) # converted to BGR\n",
    "    \n",
    "    y_pred_sharded = model.predict(x_val, verbose=0, use_multiprocessing=True, batch_size=64, callbacks=None)\n",
    "    \n",
    "    try:\n",
    "        y_pred = np.concatenate([y_pred, y_pred_sharded])\n",
    "    except ValueError:\n",
    "        y_pred = y_pred_sharded\n",
    "        \n",
    "    del x_val\n",
    "    gc.collect()\n",
    "    \n",
    "    completed_percentage = (i + 1) * 100 / len(x_val_paths)\n",
    "    if completed_percentage % 5 == 0:\n",
    "        print(\"{:5.1f}% completed.\".format(completed_percentage))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Top-1 Accuracy\n",
    "\n",
    "Compare to 0.713 from Keras documentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.71248"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_accuracy(y_val_one_hot, y_pred, k=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Top-5 Accuracy\n",
    "\n",
    "Compare to 0.900 from Keras documentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.89986"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_accuracy(y_val_one_hot, y_pred, k=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(str(path_imagenet_val_dataset / \"y_pred_VGG19.npy\"), y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}