Tarun_OpenCV_Face-Emotions-Detecor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FACE EMOTIONS DETECTOR\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Dataset link : https://drive.google.com/drive/folders/1tIQ-FFeHkB9ThT5GPPGiiKiBAUxyBQ8D?usp=sharing\n",
    "* Haarcascade file link : https://github.com/tarun36rocker/Open-contributions/tree/master/haarcascades\n",
    "* .h5 file ( you can run the already trained model if you please ) : https://github.com/tarun36rocker/Open-contributions/blob/master/model5.h5\n",
    "* .md file that i made for this project : https://github.com/tarun36rocker/Open-contributions/blob/master/Tarun_OpenCV_Face-Emotions-Detector.md"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# INFORMATION YOU MIGHT WANT TO KNOW\n",
    "\n",
    "* I have SCRAPPED THE PICTURES myself so there might be few discrepancies\n",
    "* This model is NOT THE MOST ACCURATE as there is not much data that i could work with and also because BingImageClassifier has a certain limit , i think because of the images webpage ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Used for scraping the pictures from BingImageCrawler\n",
    "'''\n",
    "from icrawler.builtin import BingImageCrawler\n",
    "for keyword in ['human face happy']:\n",
    "    crawler =  BingImageCrawler(\n",
    "        parser_threads=2,\n",
    "        downloader_threads=4,\n",
    "        storage={'root_dir': 'C:/Users/Tarun/Desktop/comp/projects/face_emotions_detector/dataset/test_set/happy'} #storage={root_dir':'C:/Users/Tarun/Desktop/comp/image1/{}'.format(keyword)}\n",
    "    )\n",
    "    crawler.crawl(keyword=keyword, max_num=10, min_size=(200, 200)) '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential #initialise the neural network as its a sequence of layers\n",
    "from keras.layers import Conv2D #import convolutional layers\n",
    "from keras.layers import MaxPooling2D #import pooling layers which have max function\n",
    "from keras.layers import Flatten #import flatten layer\n",
    "from keras.layers import Dense #import dense layer\n",
    "from keras.preprocessing.image import ImageDataGenerator #preprocesses images to prevent overfitting and enrichs our training set without increasing the number of images\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "#used for getting different variations of our data (in this case pictures) as we only have a limited dataset\n",
    "#ImageDataGenerator for training data\n",
    "train_datagen = ImageDataGenerator(rescale = 1./255,\n",
    "                                   shear_range = 0.2,\n",
    "                                   zoom_range = 0.2,\n",
    "                                   horizontal_flip = True)\n",
    "\n",
    "training_set = train_datagen.flow_from_directory('dataset/training_set',\n",
    "                                                 target_size = (64, 64),\n",
    "                                                 batch_size = 32,\n",
    "                                                 class_mode = 'categorical')\n",
    "#ImageDataGenerator for test data\n",
    "test_datagen = ImageDataGenerator(rescale = 1./255)\n",
    "test_set = test_datagen.flow_from_directory('dataset/test_set',\n",
    "                                            target_size = (64, 64),\n",
    "                                            batch_size = 32,\n",
    "                                            class_mode = 'categorical')\n",
    "\n",
    "\n",
    "num_classes = len(training_set.class_indices)#to check if our model has been connected to the right folders and checking the indexes of the different emotions\n",
    "print(training_set.class_indices) \n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "# Initialising the CNN\n",
    "classifier = Sequential() # initialising the sequential layer\n",
    "\n",
    "# Step 1 - Convolution layer\n",
    "classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu')) #32 is number of feature detectors having dimensions (3,3),input_shape is format into which images will be converted .. 3 channels (rgb) of 64 * 64 coloured pixels, relu function for non-colinearity\n",
    "\n",
    "# Step 2 - Pooling .. reducing size of the feature maps\n",
    "classifier.add(MaxPooling2D(pool_size = (2, 2))) #2,2 is used so that we dont loose information and also be precise in where features are detected\n",
    "\n",
    "# Adding a second convolutional layer of convolution layer and pooling\n",
    "classifier.add(Conv2D(32, (3, 3), activation = 'relu'))\n",
    "classifier.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "\n",
    "# Step 3 - Flattening layer\n",
    "classifier.add(Flatten())\n",
    "\n",
    "# Step 4 - Full connection / dense layers\n",
    "classifier.add(Dense(units = 128, activation = 'relu')) #no. of nodes = 128 ( in power of 2 ), number around 100 is good , relu activation function is used\n",
    "classifier.add(Dense(units = num_classes, activation = 'softmax')) # softmax activation function is used here to obtain the final output or the probabilites\n",
    "\n",
    "# Compiling the CNN\n",
    "classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy']) # adam  stochastic gradient algorithm used\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
    "# Part 2 - Fitting the CNN to the images\n",
    "\n",
    "classifier.fit_generator(training_set,\n",
    "                         steps_per_epoch = 1000,\n",
    "                         epochs = 7, #reduce number of epochs because cpu times out, originally 25\n",
    "                         validation_data = test_set)\n",
    "\n",
    "# Saving model to disk\n",
    "classifier.save('model5.h5')\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#checking if our model works on a single picture\n",
    "from keras.preprocessing import image\n",
    "from keras import models\n",
    "import numpy as np\n",
    "\n",
    "model = models.load_model('model5.h5') #loading the trained model\n",
    "test_image=image.load_img('dataset/single_pred/1.jpg',target_size = (64, 64)) #testing on a single picture\n",
    "test_image=image.img_to_array(test_image) #converts it into 3d array\n",
    "test_image=np.expand_dims(test_image,axis=0)\n",
    "prediction=model.predict([test_image]) #finding the prediction by inputting the array into the model\n",
    "\n",
    "count=0\n",
    "for i in range(len(prediction[0])):\n",
    "    if(prediction[0][i]==0):\n",
    "        count+=1\n",
    "    elif(prediction[0][i]==1):\n",
    "        break\n",
    "print(prediction[0])        \n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2 \n",
    "from keras import models\n",
    "from PIL import Image\n",
    "from keras.preprocessing import image\n",
    "\n",
    "model = models.load_model('model5.h5') #loading the model\n",
    "cap = cv2.VideoCapture(0) #getting the object from the webcam ( default webcam input is at 0)\n",
    "face_cascade = cv2.CascadeClassifier('haarcascades/haarcascade_frontalface_default.xml')#using haarcascade_frontalface to detect the face\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "#main function to detect the face and the emotions its portraying\n",
    "def detect_face(img,frame_count):\n",
    "    face_img = img.copy() #getting a copy of the frame\n",
    "    face_rects = face_cascade.detectMultiScale(face_img) #detecting the face from the frame\n",
    "    for (face_x,face_y,w,h) in face_rects:\n",
    "        roi = frame[face_y:face_y+h, face_x:face_x+w] #getting the region of interest of the face\n",
    "        roi = cv2.resize(roi, (64,64)) #resizing to suit our model\n",
    "        cv2.imwrite('frame.jpg', roi) #saving as image so as to make changing of sizes and changing to arrays easier for us\n",
    "        roi=image.load_img('frame.jpg',target_size = (64, 64))\n",
    "        img_array=image.img_to_array(roi) #converts it into 3d array\n",
    "        img_array=np.expand_dims(img_array,axis=0)\n",
    "        prediction=model.predict([img_array]) #predicting by passing array into the model\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "        count=0\n",
    "        for i in range(len(prediction[0])): #getting the predicition value from the array\n",
    "            if(prediction[0][i]==0):\n",
    "                count+=1\n",
    "            elif(prediction[0][i]==1):\n",
    "                break       \n",
    "        #Checking the predicted emotion based on our prediction        \n",
    "        #BGR COLOR CODE\n",
    "        if(count==0.0): \n",
    "            prediction=\"Angry\" \n",
    "            color=(0,0,255)\n",
    "        elif(count==1.0):\n",
    "            prediction=\"Disgust\"\n",
    "            color=(0,255,0)\n",
    "        elif(count==2.0):\n",
    "            prediction=\"Fear\"\n",
    "            color=(255,0,0)\n",
    "        elif(count==3.0):\n",
    "            prediction=\"Happy\"\n",
    "            color=(0,255,255)\n",
    "        elif(count==4.0):\n",
    "            prediction=\"Sad\"\n",
    "            color=(139,0,0)\n",
    "        else:\n",
    "            prediction=\"Dont know!\"\n",
    "            color=(156,121,135)\n",
    "            \n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------  \n",
    "        #writing our predictions on our live frame\n",
    "        font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "        cv2.rectangle(img, (face_x,face_y), (face_x+w,face_y+h), (255,255,255), 2)\n",
    "        if(frame_count%2==0):\n",
    "            cv2.putText(img,prediction,(200,130), font, 1, color, 2, cv2.LINE_AA) \n",
    "    return img\n",
    "\n",
    "#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "frame_count=0     \n",
    "while True: \n",
    "    #getting frames from webcam\n",
    "    ret, frame = cap.read(0)\n",
    "    #passing frames to the emotions detection function\n",
    "    frame = detect_face(frame,frame_count)\n",
    "    frame_count+=1\n",
    "    cv2.imshow('Emotions detection', frame)#final outputting of the frame \n",
    "    c = cv2.waitKey(1) \n",
    "    if c == 27: \n",
    "        break      \n",
    "cap.release() \n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}