Criminal Recognition/Image Search from Database Method 1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing Opencv for preprocessing\n",
    "import cv2\n",
    "\n",
    "#importing Dlib for face detction and face alignment\n",
    "import dlib\n",
    "\n",
    "#importing numpy for doing mathematical operation\n",
    "import numpy as np\n",
    "\n",
    "#importing load_model function from model module to load our saved model\n",
    "from keras.models import load_model\n",
    "\n",
    "#importing matplotlib for seeing input and output image\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#importing scipy for calculating the similarity between images\n",
    "from scipy import spatial\n",
    "\n",
    "#loading trained model on images\n",
    "model = load_model('./models/facenet_keras.h5')\n",
    "\n",
    "#creating dlib detector to detct face from input image\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "\n",
    "#creaeting dlib predcitor to predcit 68 landmarks of input image and then aligning images\n",
    "predictor = dlib.shape_predictor('./models/shape_predictor_68_face_landmarks.dat')\n",
    "                            \n",
    "#removing unnecessary warnings\n",
    "import warnings\n",
    "def fxn():\n",
    "    warnings.warn(\"deprecated\", DeprecationWarning)\n",
    "\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    fxn()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Required function for doing facial recignition and face alignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function to input in image, do it's alignment then return 128D embedding which contain important features about image which determines it's identity\n",
    "def extract_face_and_get_embedding(filename, required_size, model, i):\n",
    "    \n",
    "    #taking in input image and changing it to RGB format as in OpenCv it is in BGR format\n",
    "    img = cv2.imread(filename)\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "    \n",
    "    #detecting face from input image\n",
    "    dets = detector(img, 3)\n",
    "\n",
    "    \n",
    "    #counting number of faces in image because if there are no faces then no need to go further\n",
    "    num_faces = len(dets)\n",
    "\n",
    "    #if number of faces == 0 then we will check once again with our second model(Haar Cascade) if again no face found then output\n",
    "    #message will be given else we will go further for computation\n",
    "    if num_faces == 0:\n",
    "        \n",
    "        #using Haar Cascade for doing face detection(as backup model for face detection) if our first model is unable to detect face \n",
    "        face_cascade = cv2.CascadeClassifier('models/haarcascade_frontalface_default.xml')\n",
    "        rects = face_cascade.detectMultiScale(img, minNeighbors = 5)\n",
    "        \n",
    "        #if face found from Haarcascade we will proceed for futhur computation else we will give final messge that ace not found\n",
    "        if len(rects) != 0:\n",
    "            \n",
    "            #looping through coordinates found in input image\n",
    "            for x, y, w, h in rects:\n",
    "                \n",
    "                #extracting face from image for furthur computation\n",
    "                roi_face = img[y-5:y+h+10, x-5:x+w+10]\n",
    "                \n",
    "                #converting face image into numpy array\n",
    "                image = np.array(roi_face)\n",
    "                \n",
    "                #resizing image to 3D array as our model take input image of shape required_size x required_size x 3\n",
    "                image = cv2.resize(image, (required_size, required_size), interpolation = cv2.INTER_AREA)\n",
    "                face_pixels = image.astype('float32')\n",
    "\n",
    "                #finding mean and standard deviation\n",
    "                mean, std = face_pixels.mean(), face_pixels.std()\n",
    "                face_pixels = (face_pixels - mean) / std\n",
    "                samples = np.expand_dims(face_pixels, axis=0)\n",
    "\n",
    "                #predicting the ouput 128D embedding for input image\n",
    "                yhat = model.predict(samples)\n",
    "\n",
    "                return yhat[0]\n",
    "                \n",
    "        elif len(rects) == 0:\n",
    "            print(\"Sorry, there were no faces found in image\", i)\n",
    "            \n",
    "        else:\n",
    "             print('More than one number of faces found in image', i)\n",
    "\n",
    "    elif num_faces == 1:\n",
    "\n",
    "        faces = dlib.full_object_detections()\n",
    "\n",
    "        #predicting landmarks of detected face from input image\n",
    "        for detection in dets:\n",
    "            faces.append(predictor(img, detection))\n",
    "\n",
    "        #aligning image to get better accuracy while comparing 2 images\n",
    "        #We can play with padding to zoom-in and out to get required image\n",
    "        image = dlib.get_face_chips(img, faces, size = required_size, padding = 0.2)\n",
    "\n",
    "        #converting image to numpy array\n",
    "        image = np.array(image)\n",
    "\n",
    "        #reshaping image to 3D array as output of dlib predictor is 4D array of form 1 x required_size x required_size x 3\n",
    "        image = image.reshape(required_size, required_size, 3)\n",
    "        face_pixels = image.astype('float32')\n",
    "\n",
    "        #finding mean and standard deviation\n",
    "        mean, std = face_pixels.mean(), face_pixels.std()\n",
    "        face_pixels = (face_pixels - mean) / std\n",
    "        samples = np.expand_dims(face_pixels, axis=0)\n",
    "\n",
    "        #predicting the ouput 128D embedding for input image\n",
    "        yhat = model.predict(samples)\n",
    "\n",
    "        return yhat[0]\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        print('More than one number of faces found in image', i)\n",
    "\n",
    "#function for comparing images in database and predicting top n similar images from database\n",
    "def who_it_is(path_of_image, database, model, i):\n",
    "    \n",
    "    #storing similarity distance in dictionary 'similarity_distance' while storing name as well as distance in list 'list_of_similarity' for later sorting on basis of distance\n",
    "    similarity_distance, list_of_similarity = {}, []\n",
    "    \n",
    "    #getting 128D encoding of input image from our trained model which includes important features, identity as well as important information about input image\n",
    "    encoding = extract_face_and_get_embedding(path_of_image, 160, model, i)\n",
    "    \n",
    "    #initializing min_dist with some random larger number\n",
    "    min_dist = 10\n",
    "    \n",
    "    #if no face is found in image than we are returning flag 1 and None which will tell us that in later function that there is no face in input image\n",
    "    if encoding is None:\n",
    "        list_of_similarity.append((1, None))\n",
    "        return None, list_of_similarity\n",
    "    \n",
    "    #if face is found then we are doing furthur computation\n",
    "    else:\n",
    "        \n",
    "        #searching throughout database and comparing with each image in database\n",
    "        for (name, db_enc) in database.items():\n",
    "            \n",
    "            #if there is image in database where our model is not able to detect any face we are ignoring that image\n",
    "            if db_enc is not None:\n",
    "\n",
    "                #calculating similarity distance between images\n",
    "                dist = spatial.distance.cosine(db_enc, encoding)\n",
    "                \n",
    "                #storing distance for later use\n",
    "                similarity_distance[name] = dist\n",
    "\n",
    "                list_of_similarity.append((dist, name))\n",
    "\n",
    "                #storing image of minimum distacne in the database\n",
    "                if dist < min_dist:\n",
    "\n",
    "                    min_dist = dist\n",
    "                    identity = name\n",
    "\n",
    "        #returning similarity distance(a dictonary)\n",
    "        return similarity_distance, list_of_similarity\n",
    "\n",
    "#function to show top n similar image as well as input image and their similarity distances with their name in database\n",
    "def output(image_path, similarity_distance, list_of_similarity, number_of_images_to_show):\n",
    "    \n",
    "    #showing input image\n",
    "    plt.imshow(cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB))\n",
    "    plt.title('Input Image')\n",
    "    plt.show()\n",
    "    \n",
    "    if len(list_of_similarity) >= 2 :\n",
    "\n",
    "        #sorting similarity in ascending order to take out top n minimum similarity distances\n",
    "        list_of_similarity.sort()\n",
    "        print('\\n\\n')\n",
    "        #showing best n similar images found from databse\n",
    "        print('TOP '+str(number_of_images_to_show)+' BEST IMAGES FOUND FROM DATABASE ARE :-----')\n",
    "        for i in range(number_of_images_to_show):\n",
    "            print('\\n\\n')\n",
    "            print('Similar Image '+str(i+1)+' found is: '+str(list_of_similarity[i][1])+'.jpg with similarity '+str(round(list_of_similarity[i][0], 2)))\n",
    "            plt.imshow(cv2.cvtColor(cv2.imread('Dataset/Database/'+str(list_of_similarity[i][1])+'.jpg'), cv2.COLOR_BGR2RGB))\n",
    "            plt.show()\n",
    "            print('\\n\\n')\n",
    "            \n",
    "    else:\n",
    "        print('Sorry!, No face found in image')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sorry, there were no faces found in image  41\n",
      "Sorry, there were no faces found in image  96\n",
      "Wall time: 4min 44s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#creating embeddings of all the images in database and storing in variable 'database' to save at the time of \n",
    "#outputing the images otherwise it will take around 15 min for giving output of each image beacause it has to calculate \n",
    "#embedding each time of whole database.\n",
    "\n",
    "database = {}\n",
    "for i in range(169):\n",
    "    database[i+1] = extract_face_and_get_embedding('Dataset/Database/'+str(i+1)+'.jpg', 160, model, i+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "for i in range(1, 119):\n",
    "    image_path, image_name = 'Dataset/Test Images/'+str(i)+'.jpg', i\n",
    "    output(image_path, *who_it_is(image_path, database, model, image_name), 5)\n",
    "    print('\\n\\n\\n\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}