Handwritten Equation Solver using CNN/Equation solver CNN.ipynb

{
 "cells": [
  {
   "source": [
    "### Importing required libraries"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Flatten\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K\n",
    "#K.set_image_dim_ordering('th')\n",
    "from keras.models import model_from_json\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tf.__version__ is 2.1.0\ntf.keras.__version__ is: 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "#To avoid tensorflow keras version error set available gpus in your local system\n",
    "import tensorflow as tf\n",
    "import keras.backend.tensorflow_backend as tfback\n",
    "\n",
    "print(\"tf.__version__ is\", tf.__version__)\n",
    "print(\"tf.keras.__version__ is:\", tf.keras.__version__)\n",
    "\n",
    "def _get_available_gpus():\n",
    "    \"\"\"Get a list of available gpu devices (formatted as strings).\n",
    "\n",
    "    # Returns\n",
    "        A list of available GPU devices.\n",
    "    \"\"\"\n",
    "    #global _LOCAL_DEVICES\n",
    "    if tfback._LOCAL_DEVICES is None:\n",
    "        devices = tf.config.list_logical_devices()\n",
    "        tfback._LOCAL_DEVICES = [x.name for x in devices]\n",
    "    return [x for x in tfback._LOCAL_DEVICES if 'device:gpu' in x.lower()]\n",
    "\n",
    "tfback._get_available_gpus = _get_available_gpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load model from json file\n",
    "json_file = open('model_final.json', 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "loaded_model = model_from_json(loaded_model_json)\n",
    "# load weights into new model\n",
    "loaded_model.load_weights(\"model_final.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read the image of handwritten equation\n",
    "import cv2\n",
    "import numpy as np\n",
    "img = cv2.imread(r'C:\\Users\\hp\\Desktop\\Handwritten Equation Solver using CNN\\test_img.jpeg',cv2.IMREAD_GRAYSCALE)\n",
    "\n",
    "cv2.imshow(\"image\",img)\n",
    "cv2.waitKey(0)\n",
    "cv2.destroyAllWindows()\n",
    "\n",
    "\n",
    "if img is not None:\n",
    "    #images.append(img)\n",
    "    img=~img\n",
    "    ret,thresh=cv2.threshold(img,127,255,cv2.THRESH_BINARY)\n",
    "    ret,ctrs,ret=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n",
    "    cnt=sorted(ctrs, key=lambda ctr: cv2.boundingRect(ctr)[0])\n",
    "    w=int(28)\n",
    "    h=int(28)\n",
    "    train_data=[]\n",
    "    #print(len(cnt))\n",
    "    rects=[]\n",
    "    for c in cnt :\n",
    "        x,y,w,h= cv2.boundingRect(c)\n",
    "        rect=[x,y,w,h]\n",
    "        rects.append(rect)\n",
    "    #print(rects)\n",
    "    bool_rect=[]\n",
    "    for r in rects:\n",
    "        l=[]\n",
    "        for rec in rects:\n",
    "            flag=0\n",
    "            if rec!=r:\n",
    "                if r[0]<(rec[0]+rec[2]+10) and rec[0]<(r[0]+r[2]+10) and r[1]<(rec[1]+rec[3]+10) and rec[1]<(r[1]+r[3]+10):\n",
    "                    flag=1\n",
    "                l.append(flag)\n",
    "            if rec==r:\n",
    "                l.append(0)\n",
    "        bool_rect.append(l)\n",
    "    #print(bool_rect)\n",
    "    dump_rect=[]\n",
    "    for i in range(0,len(cnt)):\n",
    "        for j in range(0,len(cnt)):\n",
    "            if bool_rect[i][j]==1:\n",
    "                area1=rects[i][2]*rects[i][3]\n",
    "                area2=rects[j][2]*rects[j][3]\n",
    "                if(area1==min(area1,area2)):\n",
    "                    dump_rect.append(rects[i])\n",
    "    #print(len(dump_rect)) \n",
    "    final_rect=[i for i in rects if i not in dump_rect]\n",
    "    #print(final_rect)\n",
    "    for r in final_rect:\n",
    "        x=r[0]\n",
    "        y=r[1]\n",
    "        w=r[2]\n",
    "        h=r[3]\n",
    "        im_crop =thresh[y:y+h+10,x:x+w+10]\n",
    "        \n",
    "\n",
    "        im_resize = cv2.resize(im_crop,(28,28))\n",
    "        cv2.imshow(\"work\",im_resize)\n",
    "        cv2.waitKey(0)\n",
    "        cv2.destroyAllWindows()\n",
    "\n",
    "        im_resize=np.reshape(im_resize,(1,28,28))\n",
    "        train_data.append(im_resize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "82+8\n"
     ]
    }
   ],
   "source": [
    "#Recognize handwritten digits with trained model\n",
    "s=''\n",
    "for i in range(len(train_data)):\n",
    "    train_data[i]=np.array(train_data[i])\n",
    "    train_data[i]=train_data[i].reshape(1,1,28,28)\n",
    "    result=loaded_model.predict_classes(train_data[i])\n",
    "    if(result[0]==10):\n",
    "        s=s+'-'\n",
    "    if(result[0]==11):\n",
    "        s=s+'+'\n",
    "    if(result[0]==12):\n",
    "        s=s+'*'\n",
    "    if(result[0]==0):\n",
    "        s=s+'0'\n",
    "    if(result[0]==1):\n",
    "        s=s+'1'\n",
    "    if(result[0]==2):\n",
    "        s=s+'2'\n",
    "    if(result[0]==3):\n",
    "        s=s+'3'\n",
    "    if(result[0]==4):\n",
    "        s=s+'4'\n",
    "    if(result[0]==5):\n",
    "        s=s+'5'\n",
    "    if(result[0]==6):\n",
    "        s=s+'6'\n",
    "    if(result[0]==7):\n",
    "        s=s+'7'\n",
    "    if(result[0]==8):\n",
    "        s=s+'8'\n",
    "    if(result[0]==9):\n",
    "        s=s+'9'\n",
    "    \n",
    "print(s)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "90"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "#Evaluate the handwritten equation\n",
    "eval(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.10 64-bit (conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "interpreter": {
   "hash": "2e6c568e5a0f9d31b8a624b109502eabbe2643fc6d1f5989adcc8d5adbd785e0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}