deep-learning-project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary : Main file for generating poses coordinates dataset. And loading the trained model for performing pose estimation "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-11T20:55:28.410928Z",
     "iopub.status.busy": "2022-11-11T20:55:28.410324Z",
     "iopub.status.idle": "2022-11-11T20:55:28.435382Z",
     "shell.execute_reply": "2022-11-11T20:55:28.434233Z",
     "shell.execute_reply.started": "2022-11-11T20:55:28.410783Z"
    },
    "id": "QKe-ubNcDvgv"
   },
   "source": [
    "## Structure of the file\n",
    " ### 1. Code for Movenet\n",
    " \n",
    " ### 2. Code for extracting points for model training\n",
    " \n",
    " ### 3. Yoga Pose Estimation using saved model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "10_zkgbZBkIE"
   },
   "source": [
    "# 1. Code From Tensorflow Documentation with Apache 2.0 License ( Required to run Movenet )  [Start]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# =============================================================================="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apache License : http://www.apache.org/licenses/LICENSE-2.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9u_VGR6_BmbZ"
   },
   "source": [
    "## Visualization libraries & Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:51:38.881017Z",
     "iopub.status.busy": "2022-12-08T19:51:38.880498Z",
     "iopub.status.idle": "2022-12-08T19:52:17.999393Z",
     "shell.execute_reply": "2022-12-08T19:52:17.998200Z",
     "shell.execute_reply.started": "2022-12-08T19:51:38.880968Z"
    },
    "id": "TtcwSIcgbIVN"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.0.1; however, version 22.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\python.exe -m pip install --upgrade pip' command.\n",
      "WARNING: You are using pip version 21.0.1; however, version 22.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\python.exe -m pip install --upgrade pip' command.\n",
      "WARNING: You are using pip version 21.0.1; however, version 22.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install -q imageio\n",
    "!pip install -q opencv-python\n",
    "!pip install -q git+https://github.com/tensorflow/docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow_hub in c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\lib\\site-packages (0.12.0)\n",
      "Requirement already satisfied: numpy>=1.12.0 in c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\lib\\site-packages (from tensorflow_hub) (1.23.3)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\lib\\site-packages (from tensorflow_hub) (3.14.0)\n",
      "Requirement already satisfied: six>=1.9 in c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\lib\\site-packages (from protobuf>=3.8.0->tensorflow_hub) (1.14.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.0.1; however, version 22.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: PySimpleGUI in c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\lib\\site-packages (4.60.4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 21.0.1; however, version 22.3.1 is available.\n",
      "You should consider upgrading via the 'c:\\users\\mynam\\appdata\\local\\programs\\python\\python38\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install PySimpleGUI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:52:18.002462Z",
     "iopub.status.busy": "2022-12-08T19:52:18.001796Z",
     "iopub.status.idle": "2022-12-08T19:52:28.694292Z",
     "shell.execute_reply": "2022-12-08T19:52:28.693263Z",
     "shell.execute_reply.started": "2022-12-08T19:52:18.002417Z"
    },
    "id": "9BLeJv-pCCld"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "from tensorflow_docs.vis import embed\n",
    "import numpy as np\n",
    "import cv2\n",
    "\n",
    "# Import matplotlib libraries\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.collections import LineCollection\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "# Some modules to display an animation using imageio.\n",
    "import imageio\n",
    "from IPython.display import HTML, display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:52:28.696491Z",
     "iopub.status.busy": "2022-12-08T19:52:28.695806Z",
     "iopub.status.idle": "2022-12-08T19:52:28.727275Z",
     "shell.execute_reply": "2022-12-08T19:52:28.724711Z",
     "shell.execute_reply.started": "2022-12-08T19:52:28.696450Z"
    },
    "id": "bEJBMeRb3YUy"
   },
   "outputs": [],
   "source": [
    "#@title Helper functions for visualization\n",
    "\n",
    "# Dictionary that maps from joint names to keypoint indices.\n",
    "KEYPOINT_DICT = {\n",
    "    'nose': 0,\n",
    "    'left_eye': 1,\n",
    "    'right_eye': 2,\n",
    "    'left_ear': 3,\n",
    "    'right_ear': 4,\n",
    "    'left_shoulder': 5,\n",
    "    'right_shoulder': 6,\n",
    "    'left_elbow': 7,\n",
    "    'right_elbow': 8,\n",
    "    'left_wrist': 9,\n",
    "    'right_wrist': 10,\n",
    "    'left_hip': 11,\n",
    "    'right_hip': 12,\n",
    "    'left_knee': 13,\n",
    "    'right_knee': 14,\n",
    "    'left_ankle': 15,\n",
    "    'right_ankle': 16\n",
    "}\n",
    "\n",
    "# Maps bones to a matplotlib color name.\n",
    "KEYPOINT_EDGE_INDS_TO_COLOR = {\n",
    "    (0, 1): 'm',\n",
    "    (0, 2): 'c',\n",
    "    (1, 3): 'm',\n",
    "    (2, 4): 'c',\n",
    "    (0, 5): 'm',\n",
    "    (0, 6): 'c',\n",
    "    (5, 7): 'm',\n",
    "    (7, 9): 'm',\n",
    "    (6, 8): 'c',\n",
    "    (8, 10): 'c',\n",
    "    (5, 6): 'y',\n",
    "    (5, 11): 'm',\n",
    "    (6, 12): 'c',\n",
    "    (11, 12): 'y',\n",
    "    (11, 13): 'm',\n",
    "    (13, 15): 'm',\n",
    "    (12, 14): 'c',\n",
    "    (14, 16): 'c'\n",
    "}\n",
    "\n",
    "def _keypoints_and_edges_for_display(keypoints_with_scores,\n",
    "                                     height,\n",
    "                                     width,\n",
    "                                     keypoint_threshold=0.11):\n",
    "  \"\"\"Returns high confidence keypoints and edges for visualization.\n",
    "\n",
    "  Args:\n",
    "    keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
    "      the keypoint coordinates and scores returned from the MoveNet model.\n",
    "    height: height of the image in pixels.\n",
    "    width: width of the image in pixels.\n",
    "    keypoint_threshold: minimum confidence score for a keypoint to be\n",
    "      visualized.\n",
    "\n",
    "  Returns:\n",
    "    A (keypoints_xy, edges_xy, edge_colors) containing:\n",
    "      * the coordinates of all keypoints of all detected entities;\n",
    "      * the coordinates of all skeleton edges of all detected entities;\n",
    "      * the colors in which the edges should be plotted.\n",
    "  \"\"\"\n",
    "  keypoints_all = []\n",
    "  keypoint_edges_all = []\n",
    "  edge_colors = []\n",
    "  num_instances, _, _, _ = keypoints_with_scores.shape\n",
    "  for idx in range(num_instances):\n",
    "    kpts_x = keypoints_with_scores[0, idx, :, 1]\n",
    "    kpts_y = keypoints_with_scores[0, idx, :, 0]\n",
    "    kpts_scores = keypoints_with_scores[0, idx, :, 2]\n",
    "    kpts_absolute_xy = np.stack(\n",
    "        [width * np.array(kpts_x), height * np.array(kpts_y)], axis=-1)\n",
    "    kpts_above_thresh_absolute = kpts_absolute_xy[\n",
    "        kpts_scores > keypoint_threshold, :]\n",
    "    keypoints_all.append(kpts_above_thresh_absolute)\n",
    "\n",
    "    for edge_pair, color in KEYPOINT_EDGE_INDS_TO_COLOR.items():\n",
    "      if (kpts_scores[edge_pair[0]] > keypoint_threshold and\n",
    "          kpts_scores[edge_pair[1]] > keypoint_threshold):\n",
    "        x_start = kpts_absolute_xy[edge_pair[0], 0]\n",
    "        y_start = kpts_absolute_xy[edge_pair[0], 1]\n",
    "        x_end = kpts_absolute_xy[edge_pair[1], 0]\n",
    "        y_end = kpts_absolute_xy[edge_pair[1], 1]\n",
    "        line_seg = np.array([[x_start, y_start], [x_end, y_end]])\n",
    "        keypoint_edges_all.append(line_seg)\n",
    "        edge_colors.append(color)\n",
    "  if keypoints_all:\n",
    "    keypoints_xy = np.concatenate(keypoints_all, axis=0)\n",
    "  else:\n",
    "    keypoints_xy = np.zeros((0, 17, 2))\n",
    "\n",
    "  if keypoint_edges_all:\n",
    "    edges_xy = np.stack(keypoint_edges_all, axis=0)\n",
    "  else:\n",
    "    edges_xy = np.zeros((0, 2, 2))\n",
    "  return keypoints_xy, edges_xy, edge_colors\n",
    "\n",
    "\n",
    "def draw_prediction_on_image(\n",
    "    image, keypoints_with_scores, crop_region=None, close_figure=False,\n",
    "    output_image_height=None):\n",
    "  \"\"\"Draws the keypoint predictions on image.\n",
    "\n",
    "  Args:\n",
    "    image: A numpy array with shape [height, width, channel] representing the\n",
    "      pixel values of the input image.\n",
    "    keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
    "      the keypoint coordinates and scores returned from the MoveNet model.\n",
    "    crop_region: A dictionary that defines the coordinates of the bounding box\n",
    "      of the crop region in normalized coordinates (see the init_crop_region\n",
    "      function below for more detail). If provided, this function will also\n",
    "      draw the bounding box on the image.\n",
    "    output_image_height: An integer indicating the height of the output image.\n",
    "      Note that the image aspect ratio will be the same as the input image.\n",
    "\n",
    "  Returns:\n",
    "    A numpy array with shape [out_height, out_width, channel] representing the\n",
    "    image overlaid with keypoint predictions.\n",
    "  \"\"\"\n",
    "  height, width, channel = image.shape\n",
    "  aspect_ratio = float(width) / height\n",
    "  fig, ax = plt.subplots(figsize=(12 * aspect_ratio, 12))\n",
    "  # To remove the huge white borders\n",
    "  fig.tight_layout(pad=0)\n",
    "  ax.margins(0)\n",
    "  ax.set_yticklabels([])\n",
    "  ax.set_xticklabels([])\n",
    "  plt.axis('off')\n",
    "\n",
    "  im = ax.imshow(image)\n",
    "  line_segments = LineCollection([], linewidths=(4), linestyle='solid')\n",
    "  ax.add_collection(line_segments)\n",
    "  # Turn off tick labels\n",
    "  scat = ax.scatter([], [], s=60, color='#FF1493', zorder=3)\n",
    "\n",
    "  (keypoint_locs, keypoint_edges,\n",
    "   edge_colors) = _keypoints_and_edges_for_display(\n",
    "       keypoints_with_scores, height, width)\n",
    "\n",
    "  line_segments.set_segments(keypoint_edges)\n",
    "  line_segments.set_color(edge_colors)\n",
    "  if keypoint_edges.shape[0]:\n",
    "    line_segments.set_segments(keypoint_edges)\n",
    "    line_segments.set_color(edge_colors)\n",
    "  if keypoint_locs.shape[0]:\n",
    "    scat.set_offsets(keypoint_locs)\n",
    "\n",
    "  if crop_region is not None:\n",
    "    xmin = max(crop_region['x_min'] * width, 0.0)\n",
    "    ymin = max(crop_region['y_min'] * height, 0.0)\n",
    "    rec_width = min(crop_region['x_max'], 0.99) * width - xmin\n",
    "    rec_height = min(crop_region['y_max'], 0.99) * height - ymin\n",
    "    rect = patches.Rectangle(\n",
    "        (xmin,ymin),rec_width,rec_height,\n",
    "        linewidth=1,edgecolor='b',facecolor='none')\n",
    "    ax.add_patch(rect)\n",
    "\n",
    "  fig.canvas.draw()\n",
    "  image_from_plot = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)\n",
    "  image_from_plot = image_from_plot.reshape(\n",
    "      fig.canvas.get_width_height()[::-1] + (3,))\n",
    "  plt.close(fig)\n",
    "  if output_image_height is not None:\n",
    "    output_image_width = int(output_image_height / height * width)\n",
    "    image_from_plot = cv2.resize(\n",
    "        image_from_plot, dsize=(output_image_width, output_image_height),\n",
    "         interpolation=cv2.INTER_CUBIC)\n",
    "  return image_from_plot\n",
    "\n",
    "def to_gif(images, fps):\n",
    "  \"\"\"Converts image sequence (4D numpy array) to gif.\"\"\"\n",
    "  imageio.mimsave('./animation.gif', images, fps=fps)\n",
    "  return embed.embed_file('./animation.gif')\n",
    "\n",
    "def progress(value, max=100):\n",
    "  return HTML(\"\"\"\n",
    "      <progress\n",
    "          value='{value}'\n",
    "          max='{max}',\n",
    "          style='width: 100%'\n",
    "      >\n",
    "          {value}\n",
    "      </progress>\n",
    "  \"\"\".format(value=value, max=max))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UvrN0iQiOxhR"
   },
   "source": [
    "## Load Model from TF hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:52:28.731571Z",
     "iopub.status.busy": "2022-12-08T19:52:28.730496Z",
     "iopub.status.idle": "2022-12-08T19:52:49.374436Z",
     "shell.execute_reply": "2022-12-08T19:52:49.373392Z",
     "shell.execute_reply.started": "2022-12-08T19:52:28.731494Z"
    },
    "id": "zeGHgANcT7a1"
   },
   "outputs": [],
   "source": [
    "model_name = \"movenet_lightning\" #@param [\"movenet_lightning\", \"movenet_thunder\", \"movenet_lightning_f16.tflite\", \"movenet_thunder_f16.tflite\", \"movenet_lightning_int8.tflite\", \"movenet_thunder_int8.tflite\"]\n",
    "\n",
    "if \"tflite\" in model_name:\n",
    "  if \"movenet_lightning_f16\" in model_name:\n",
    "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/tflite/float16/4?lite-format=tflite\n",
    "    input_size = 192\n",
    "  elif \"movenet_thunder_f16\" in model_name:\n",
    "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/thunder/tflite/float16/4?lite-format=tflite\n",
    "    input_size = 256\n",
    "  elif \"movenet_lightning_int8\" in model_name:\n",
    "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/tflite/int8/4?lite-format=tflite\n",
    "    input_size = 192\n",
    "  elif \"movenet_thunder_int8\" in model_name:\n",
    "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/thunder/tflite/int8/4?lite-format=tflite\n",
    "    input_size = 256\n",
    "  else:\n",
    "    raise ValueError(\"Unsupported model name: %s\" % model_name)\n",
    "\n",
    "  # Initialize the TFLite interpreter\n",
    "  interpreter = tf.lite.Interpreter(model_path=\"model.tflite\")\n",
    "  interpreter.allocate_tensors()\n",
    "\n",
    "  def movenet(input_image):\n",
    "    \"\"\"Runs detection on an input image.\n",
    "\n",
    "    Args:\n",
    "      input_image: A [1, height, width, 3] tensor represents the input image\n",
    "        pixels. Note that the height/width should already be resized and match the\n",
    "        expected input resolution of the model before passing into this function.\n",
    "\n",
    "    Returns:\n",
    "      A [1, 1, 17, 3] float numpy array representing the predicted keypoint\n",
    "      coordinates and scores.\n",
    "    \"\"\"\n",
    "    # TF Lite format expects tensor type of uint8.\n",
    "    input_image = tf.cast(input_image, dtype=tf.uint8)\n",
    "    input_details = interpreter.get_input_details()\n",
    "    output_details = interpreter.get_output_details()\n",
    "    interpreter.set_tensor(input_details[0]['index'], input_image.numpy())\n",
    "    # Invoke inference.\n",
    "    interpreter.invoke()\n",
    "    # Get the model prediction.\n",
    "    keypoints_with_scores = interpreter.get_tensor(output_details[0]['index'])\n",
    "    return keypoints_with_scores\n",
    "\n",
    "else:\n",
    "  if \"movenet_lightning\" in model_name:\n",
    "    module = hub.load(\"https://tfhub.dev/google/movenet/singlepose/lightning/4\")\n",
    "    input_size = 192\n",
    "  elif \"movenet_thunder\" in model_name:\n",
    "    module = hub.load(\"https://tfhub.dev/google/movenet/singlepose/thunder/4\")\n",
    "    input_size = 256\n",
    "  else:\n",
    "    raise ValueError(\"Unsupported model name: %s\" % model_name)\n",
    "\n",
    "  def movenet(input_image):\n",
    "    \"\"\"Runs detection on an input image.\n",
    "\n",
    "    Args:\n",
    "      input_image: A [1, height, width, 3] tensor represents the input image\n",
    "        pixels. Note that the height/width should already be resized and match the\n",
    "        expected input resolution of the model before passing into this function.\n",
    "\n",
    "    Returns:\n",
    "      A [1, 1, 17, 3] float numpy array representing the predicted keypoint\n",
    "      coordinates and scores.\n",
    "    \"\"\"\n",
    "    model = module.signatures['serving_default']\n",
    "\n",
    "    # SavedModel format expects tensor type of int32.\n",
    "    input_image = tf.cast(input_image, dtype=tf.int32)\n",
    "    # Run model inference.\n",
    "    outputs = model(input_image)\n",
    "    # Output is a [1, 1, 17, 3] tensor.\n",
    "    keypoints_with_scores = outputs['output_0'].numpy()\n",
    "    return keypoints_with_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow Movenet Code [End]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Code for loading Yoga dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:53:04.383990Z",
     "iopub.status.busy": "2022-12-08T19:53:04.383219Z",
     "iopub.status.idle": "2022-12-08T19:53:04.430751Z",
     "shell.execute_reply": "2022-12-08T19:53:04.429658Z",
     "shell.execute_reply.started": "2022-12-08T19:53:04.383948Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import imageio as iio\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "import pandas as pd\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:53:04.436385Z",
     "iopub.status.busy": "2022-12-08T19:53:04.435892Z",
     "iopub.status.idle": "2022-12-08T19:53:04.443757Z",
     "shell.execute_reply": "2022-12-08T19:53:04.442673Z",
     "shell.execute_reply.started": "2022-12-08T19:53:04.436354Z"
    }
   },
   "outputs": [],
   "source": [
    "#Define all channels because images have different channels in some cases and different in other\n",
    "all_channels = ['RGBA','CMYK', 'RGBA']\n",
    "def image_reader_folder(path):\n",
    "    images_list = os.listdir(path)\n",
    "    image_read_list = []\n",
    "    for image in images_list:\n",
    "        image_read = cv2.imread(path+\"/\"+image)\n",
    "        image_read_list.append(image_read)\n",
    "    return image_read_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:53:04.446146Z",
     "iopub.status.busy": "2022-12-08T19:53:04.445320Z",
     "iopub.status.idle": "2022-12-08T19:53:04.457908Z",
     "shell.execute_reply": "2022-12-08T19:53:04.456669Z",
     "shell.execute_reply.started": "2022-12-08T19:53:04.446105Z"
    }
   },
   "outputs": [],
   "source": [
    "#Detecting yoga poses here\n",
    "def pose_detector(image, label):\n",
    "    input_image = tf.expand_dims(image, axis=0)\n",
    "    input_image = tf.image.resize_with_pad(input_image, input_size, input_size)\n",
    "    keypoints_with_scores = movenet(input_image)\n",
    "    feature_list = []\n",
    "    for feature in keypoints_with_scores.squeeze():\n",
    "        feature_list.append(feature[0])\n",
    "        feature_list.append(feature[1])\n",
    "    feature_list.append(label)\n",
    "    return feature_list\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:53:04.460485Z",
     "iopub.status.busy": "2022-12-08T19:53:04.459799Z",
     "iopub.status.idle": "2022-12-08T19:53:33.042338Z",
     "shell.execute_reply": "2022-12-08T19:53:33.041330Z",
     "shell.execute_reply.started": "2022-12-08T19:53:04.460447Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 97/97 [00:02<00:00, 39.61it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 80/80 [00:01<00:00, 44.86it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 115/115 [00:02<00:00, 50.83it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 69/69 [00:01<00:00, 45.02it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 109/109 [00:02<00:00, 50.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Images with errors\n",
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#In order to use this code please replace the location of the below location from the dataset downloaded from kaggle\n",
    "#Kaggle Link for dataset - https://www.kaggle.com/datasets/niharika41298/yoga-poses-dataset\n",
    "list_coords = []\n",
    "path_images_test = 'DATASET/TEST/'\n",
    "path_images_train = 'DATASET/TRAIN/'\n",
    "pose_names = ['downdog', 'goddess', 'plank', 'tree', 'warrior2']\n",
    "\n",
    "count = 0 \n",
    "for pose in pose_names:\n",
    "    path = path_images_test + pose\n",
    "    image_list = image_reader_folder(path)\n",
    "    for image in tqdm(image_list):\n",
    "        try:\n",
    "            list_coords.append(pose_detector(image, pose))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print(image)\n",
    "            count+=1\n",
    "            continue \n",
    "print(\"Images with errors\")\n",
    "print(count)\n",
    "#temp[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:53:33.044510Z",
     "iopub.status.busy": "2022-12-08T19:53:33.043798Z",
     "iopub.status.idle": "2022-12-08T19:53:33.050772Z",
     "shell.execute_reply": "2022-12-08T19:53:33.049668Z",
     "shell.execute_reply.started": "2022-12-08T19:53:33.044466Z"
    }
   },
   "outputs": [],
   "source": [
    "#Defnining headers here with their key value pairs for identifying joings in human body\n",
    "headers = {'nose': 0,\n",
    "    'left_eye': 1,\n",
    "    'right_eye': 2,\n",
    "    'left_ear': 3,\n",
    "    'right_ear': 4,\n",
    "    'left_shoulder': 5,\n",
    "    'right_shoulder': 6,\n",
    "    'left_elbow': 7,\n",
    "    'right_elbow': 8,\n",
    "    'left_wrist': 9,\n",
    "    'right_wrist': 10,\n",
    "    'left_hip': 11,\n",
    "    'right_hip': 12,\n",
    "    'left_knee': 13,\n",
    "    'right_knee': 14,\n",
    "    'left_ankle': 15,\n",
    "    'right_ankle': 16 \n",
    "            }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:53:33.052984Z",
     "iopub.status.busy": "2022-12-08T19:53:33.052272Z",
     "iopub.status.idle": "2022-12-08T19:53:33.084365Z",
     "shell.execute_reply": "2022-12-08T19:53:33.083296Z",
     "shell.execute_reply.started": "2022-12-08T19:53:33.052944Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "col_names = []\n",
    "for x in headers.keys():\n",
    "    col_names.append(x + '_x')\n",
    "    col_names.append(x + '_y')\n",
    "col_names.append('target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T19:53:33.086273Z",
     "iopub.status.busy": "2022-12-08T19:53:33.085910Z",
     "iopub.status.idle": "2022-12-08T19:53:33.126050Z",
     "shell.execute_reply": "2022-12-08T19:53:33.125077Z",
     "shell.execute_reply.started": "2022-12-08T19:53:33.086235Z"
    }
   },
   "outputs": [],
   "source": [
    "#col_names = []\n",
    "df_final = pd.DataFrame(list_coords, columns = col_names)\n",
    "df_final.head()\n",
    "df_final.to_csv('dataset.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note : Once 'dataset.csv' file is generated please run the file 'custom_model_'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Yoga Pose Estimation using saved model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "# first neural network with keras make predictions\n",
    "from numpy import loadtxt\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from keras.optimizers import SGD, Adam, Adadelta, RMSprop\n",
    "from keras.models import model_from_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T07:58:10.483470Z",
     "iopub.status.busy": "2022-12-08T07:58:10.483070Z",
     "iopub.status.idle": "2022-12-08T07:58:10.489495Z",
     "shell.execute_reply": "2022-12-08T07:58:10.488433Z",
     "shell.execute_reply.started": "2022-12-08T07:58:10.483438Z"
    }
   },
   "outputs": [],
   "source": [
    "# manually calculating cosine to get the value with coordinates\n",
    "def cosine(x1, y1, x0, y0, x2, y2):\n",
    "    \n",
    "    #shifting origin to get the vector\n",
    "    x_1, y_1 = x1 - x0, y1 - y0\n",
    "    x_2, y_2 = x2 - x0, y2 - y0\n",
    "    \n",
    "    #returning cosine\n",
    "    return (x_2*x_1) + (y_2*y_1) / ((x_1)**2 + (y_1)**2)**0.5 * ((x_2)**2 + (y_2)**2)**0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model from disk\n"
     ]
    }
   ],
   "source": [
    "# loading the prediction model\n",
    "json_model = 'model.json'\n",
    "model_h5 = 'model.h5'\n",
    "json_file = open(json_model, 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "loaded_model = model_from_json(loaded_model_json)\n",
    "# load weights into new model\n",
    "loaded_model.load_weights(model_h5)\n",
    "print(\"Loaded model from disk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_model.compile(Adam(lr = 0.01), \"sparse_categorical_crossentropy\", metrics = [\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_pose_names = ['Downdog', 'Goddess', 'Plank', 'Tree', 'Warrior']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imutils import resize\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-08T20:17:28.248300Z",
     "iopub.status.busy": "2022-12-08T20:17:28.247937Z",
     "iopub.status.idle": "2022-12-08T20:17:28.314092Z",
     "shell.execute_reply": "2022-12-08T20:17:28.312666Z",
     "shell.execute_reply.started": "2022-12-08T20:17:28.248269Z"
    }
   },
   "outputs": [],
   "source": [
    "def WebCam(chosen):\n",
    "    # capture video from webcam\n",
    "    capture = cv2.VideoCapture(0)\n",
    "    \n",
    "    # run a loop to get all the frames\n",
    "    while capture.isOpened():\n",
    "        \n",
    "        # reading the frames one by one\n",
    "        ret, frame = capture.read()\n",
    "        \n",
    "        # prepping the frame for input in the model\n",
    "        input_image = tf.expand_dims(frame.copy(), axis=0)\n",
    "        input_image = tf.image.resize_with_pad(input_image, input_size, input_size)\n",
    "        \n",
    "        # running the input through movenet to get coordinates of joints\n",
    "        keypoints_with_scores = movenet(input_image)\n",
    "        \n",
    "        feature_list = []\n",
    "        # to get x and y of each joint separately\n",
    "        for feature in keypoints_with_scores.squeeze()[5:]:\n",
    "            feature_list.append(feature[0])\n",
    "            feature_list.append(feature[1])\n",
    "        # making it a dataframe for input in the classification model\n",
    "        input_model = pd.DataFrame(feature_list).T\n",
    "        \n",
    "        # getting the index of the max probability and then using it to get the detected pose\n",
    "        detected_pose = target_pose_names[np.argmax(loaded_model.predict(input_model))]\n",
    "        \n",
    "        # putting the detected pose as text on the video display\n",
    "        cv2.putText(frame, 'Detected Yoga Pose:', (350, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.85, (255, 255, 255), 2, cv2.LINE_8)\n",
    "        cv2.putText(frame, detected_pose, (450, 60), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_4)\n",
    "        \n",
    "        \n",
    "        cv2.putText(frame, \"Press 'q' to quit.\",  (10, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2, cv2.LINE_8)\n",
    "        \n",
    "        # if the option chosen is not manual, then constantly checking when the chosen pose is detected\n",
    "        if chosen != 'Manual':\n",
    "            # keeping the original text color as red\n",
    "            color = (0, 0, 255)\n",
    "            skip = False\n",
    "            \n",
    "            # if the detected pose is chosen pose, changing the color to green\n",
    "            if chosen == detected_pose:\n",
    "                color = (0, 255, 0)\n",
    "                skip = True  # this variable is here for checking if downdog is selected and achieved\n",
    "            cv2.putText(frame, 'Goal: '+str(chosen), (350, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.85, color, 2, cv2.LINE_8)\n",
    "            \n",
    "            # if downdog is selected then this feedback loop runs\n",
    "            # if downdod is achieved then it doesn't go inside this loop\n",
    "            if chosen == 'Downdog' and not(skip):\n",
    "                feedback = 'Downward dog in progress'\n",
    "                \n",
    "                # first checking if the coordinates of shoulders are above the hip\n",
    "                if feature_list[1] < feature_list[9]:\n",
    "                    feedback = 'Start by bending down'\n",
    "                    \n",
    "                # then checking if the hip angle resides within the threshold range\n",
    "                elif cosine(feature_list[6], feature_list[7], \n",
    "                            feature_list[14], feature_list[15], \n",
    "                            feature_list[18], feature_list[19]) < 0.04:\n",
    "                    feedback = 'Bring hands closer to feet'\n",
    "                elif cosine(feature_list[6], feature_list[7], \n",
    "                            feature_list[14], feature_list[15], \n",
    "                            feature_list[18], feature_list[19]) < 0.09:\n",
    "                    feedback = 'Put hands further away from feet'\n",
    "                \n",
    "                # putting the feedback text \n",
    "                cv2.putText(frame, feedback, (350, 200), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 0, 0), 2, cv2.LINE_8)\n",
    "        \n",
    "        # displaying the joints on the image \n",
    "        display_image = tf.expand_dims(frame, axis=0)\n",
    "        display_image = tf.cast(tf.image.resize_with_pad(display_image, 1280, 1280), dtype=tf.int32)\n",
    "        output_overlay = draw_prediction_on_image(np.squeeze(display_image.numpy(), axis=0), keypoints_with_scores)\n",
    "        \n",
    "        # displaying live footage with all the overlays and texts\n",
    "        cv2.imshow('Webcam footage', output_overlay)\n",
    "        \n",
    "        # check to quit the webcam window\n",
    "        if cv2.waitKey(10)& 0xFF==ord('q'):\n",
    "            break\n",
    "    capture.release()\n",
    "\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PySimpleGUI as sg\n",
    "\n",
    "# using PySimpleGUI to get input from the user\n",
    "event, values = sg.Window('Choose an option', [[sg.Text('Yoga Poses ->'), sg.Listbox(target_pose_names + ['Manual'] , size=(20, 6), key='LB')],\n",
    "    [sg.Button('OK'), sg.Button('Cancel')]]).read(close=True)\n",
    "\n",
    "if event == 'OK':\n",
    "    try:\n",
    "        chosen = values[\"LB\"][0]\n",
    "    except:\n",
    "        chosen = 'Manual'\n",
    "    WebCam(chosen)\n",
    "    \n",
    "else:\n",
    "    sg.popup_cancel('User aborted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}