Render recognized hand gesture (text).

TheJLifeX · Feb 1, 2020 · acc06fe · acc06fe
1 parent 8c7a899
commit acc06fe
Show file tree

Hide file tree

Showing 11 changed files with 66 additions and 14 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ bazel-testlogs
 mediapipe/MediaPipe.xcodeproj
 mediapipe/MediaPipe.tulsiproj/*.tulsiconf-user
 bazel-*
+.vscode
diff --git a/hand-gesture-recognition/hand-gesture-recognition-calculator.cc b/hand-gesture-recognition/hand-gesture-recognition-calculator.cc
@@ -10,6 +10,7 @@ namespace
 {
 constexpr char normRectTag[] = "NORM_RECT";
 constexpr char normalizedLandmarkListTag[] = "NORM_LANDMARKS";
+constexpr char recognizedHandGestureTag[] = "RECOGNIZED_HAND_GESTURE";
 } // namespace
 
 // Graph config:
@@ -52,6 +53,9 @@ ::mediapipe::Status HandGestureRecognitionCalculator::GetContract(
     RET_CHECK(cc->Inputs().HasTag(normRectTag));
     cc->Inputs().Tag(normRectTag).Set<NormalizedRect>();
 
+    RET_CHECK(cc->Outputs().HasTag(recognizedHandGestureTag));
+    cc->Outputs().Tag(recognizedHandGestureTag).Set<std::string>();
+
     return ::mediapipe::OkStatus();
 }
 
@@ -65,14 +69,20 @@ ::mediapipe::Status HandGestureRecognitionCalculator::Open(
 ::mediapipe::Status HandGestureRecognitionCalculator::Process(
     CalculatorContext *cc)
 {
+    std::string *recognized_hand_gesture;
+
     // hand closed (red) rectangle
     const auto rect = &(cc->Inputs().Tag(normRectTag).Get<NormalizedRect>());
     float width = rect->width();
     float height = rect->height();
 
     if (width < 0.01 || height < 0.01)
     {
-        LOG(INFO) << "No Hand Detected";
+        // LOG(INFO) << "No Hand Detected";
+        recognized_hand_gesture = new std::string("___");
+        cc->Outputs()
+            .Tag(recognizedHandGestureTag)
+            .Add(recognized_hand_gesture, cc->InputTimestamp());
         return ::mediapipe::OkStatus();
     }
 
@@ -122,49 +132,54 @@ ::mediapipe::Status HandGestureRecognitionCalculator::Process(
     // Hand gesture recognition
     if (thumbIsOpen && firstFingerIsOpen && secondFingerIsOpen && thirdFingerIsOpen && fourthFingerIsOpen)
     {
-        LOG(INFO) << "FIVE!";
+        recognized_hand_gesture = new std::string("FIVE");
     }
     else if (!thumbIsOpen && firstFingerIsOpen && secondFingerIsOpen && thirdFingerIsOpen && fourthFingerIsOpen)
     {
-        LOG(INFO) << "FOUR!";
+        recognized_hand_gesture = new std::string("FOUR");
     }
     else if (thumbIsOpen && firstFingerIsOpen && secondFingerIsOpen && !thirdFingerIsOpen && !fourthFingerIsOpen)
     {
-        LOG(INFO) << "TREE!";
+        recognized_hand_gesture = new std::string("TREE");
     }
     else if (thumbIsOpen && firstFingerIsOpen && !secondFingerIsOpen && !thirdFingerIsOpen && !fourthFingerIsOpen)
     {
-        LOG(INFO) << "TWO!";
+        recognized_hand_gesture = new std::string("TWO");
     }
     else if (!thumbIsOpen && firstFingerIsOpen && !secondFingerIsOpen && !thirdFingerIsOpen && !fourthFingerIsOpen)
     {
-        LOG(INFO) << "ONE!";
+        recognized_hand_gesture = new std::string("ONE");
     }
     else if (!thumbIsOpen && firstFingerIsOpen && secondFingerIsOpen && !thirdFingerIsOpen && !fourthFingerIsOpen)
     {
-        LOG(INFO) << "YEAH!";
+        recognized_hand_gesture = new std::string("YEAH");
     }
     else if (!thumbIsOpen && firstFingerIsOpen && !secondFingerIsOpen && !thirdFingerIsOpen && fourthFingerIsOpen)
     {
-        LOG(INFO) << "ROCK!";
+        recognized_hand_gesture = new std::string("ROCK");
     }
     else if (thumbIsOpen && firstFingerIsOpen && !secondFingerIsOpen && !thirdFingerIsOpen && fourthFingerIsOpen)
     {
-        LOG(INFO) << "SPIDERMAN!";
+        recognized_hand_gesture = new std::string("SPIDERMAN");
     }
     else if (!thumbIsOpen && !firstFingerIsOpen && !secondFingerIsOpen && !thirdFingerIsOpen && !fourthFingerIsOpen)
     {
-        LOG(INFO) << "FIST!";
+        recognized_hand_gesture = new std::string("FIST");
     }
     else if (!firstFingerIsOpen && secondFingerIsOpen && thirdFingerIsOpen && fourthFingerIsOpen && this->isThumbNearFirstFinger(landmarkList.landmark(4), landmarkList.landmark(8)))
     {
-        LOG(INFO) << "OK!";
+        recognized_hand_gesture = new std::string("OK");
     }
     else
     {
-        LOG(INFO) << "Finger States: " << thumbIsOpen << firstFingerIsOpen << secondFingerIsOpen << thirdFingerIsOpen << fourthFingerIsOpen;
-        LOG(INFO) << "___";
+        recognized_hand_gesture = new std::string("___");
+        LOG(INFO) << "Finger States: " << thumbIsOpen << firstFingerIsOpen << secondFingerIsOpen << thirdFingerIsOpen << fourthFingerIsOpen;       
     }
+    // LOG(INFO) << recognized_hand_gesture;
+
+    cc->Outputs()
+        .Tag(recognizedHandGestureTag)
+        .Add(recognized_hand_gesture, cc->InputTimestamp());
 
     return ::mediapipe::OkStatus();
 } // namespace mediapipe

diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc
@@ -46,6 +46,7 @@ constexpr char kInputVectorTag[] = "VECTOR";
 
 constexpr char kInputFrameTagGpu[] = "INPUT_FRAME_GPU";
 constexpr char kOutputFrameTagGpu[] = "OUTPUT_FRAME_GPU";
+constexpr char recognizedHandGestureTag[] = "RECOGNIZED_HAND_GESTURE";
 
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 
@@ -165,6 +166,10 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
 
 ::mediapipe::Status AnnotationOverlayCalculator::GetContract(
     CalculatorContract* cc) {
+
+  RET_CHECK(cc->Inputs().HasTag(recognizedHandGestureTag));
+  cc->Inputs().Tag(recognizedHandGestureTag).Set<std::string>();
+
   CHECK_GE(cc->Inputs().NumEntries(), 1);
 
   bool use_gpu = false;
@@ -269,7 +274,7 @@ ::mediapipe::Status AnnotationOverlayCalculator::Open(CalculatorContext* cc) {
 }
 
 ::mediapipe::Status AnnotationOverlayCalculator::Process(
-    CalculatorContext* cc) {
+  CalculatorContext* cc) {
   // Initialize render target, drawn with OpenCV.
   std::unique_ptr<cv::Mat> image_mat;
   ImageFormat::Format target_format;
@@ -317,6 +322,9 @@ ::mediapipe::Status AnnotationOverlayCalculator::Process(
     }
   }
 
+  const auto &recognizedHandGesture = cc->Inputs().Tag(recognizedHandGestureTag).Get<std::string>();
+  renderer_->DrawText(recognizedHandGesture);
+
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     // Overlay rendered image in OpenGL, onto a copy of input.

diff --git a/mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt b/mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt
@@ -56,6 +56,7 @@ node {
   output_stream: "LANDMARKS:hand_landmarks"
   output_stream: "NORM_RECT:hand_rect_from_landmarks"
   output_stream: "PRESENCE:hand_presence"
+  output_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 }
 
 # Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
@@ -98,6 +99,7 @@ node {
   input_stream: "LANDMARKS:hand_landmarks"
   input_stream: "NORM_RECT:hand_rect"
   input_stream: "DETECTIONS:palm_detections"
+  input_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
   output_stream: "IMAGE:output_video"
 }
 
diff --git a/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt b/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
@@ -77,6 +77,7 @@ node {
   output_stream: "LANDMARKS:hand_landmarks"
   output_stream: "NORM_RECT:hand_rect_from_landmarks"
   output_stream: "PRESENCE:hand_presence"
+  output_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 }
 
 # Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
@@ -119,5 +120,6 @@ node {
   input_stream: "LANDMARKS:hand_landmarks"
   input_stream: "NORM_RECT:hand_rect"
   input_stream: "DETECTIONS:palm_detections"
+  input_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
   output_stream: "IMAGE:output_video"
 }
diff --git a/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt
@@ -7,6 +7,7 @@ input_stream: "NORM_RECT:hand_rect"
 output_stream: "LANDMARKS:hand_landmarks"
 output_stream: "NORM_RECT:hand_rect_for_next_frame"
 output_stream: "PRESENCE:hand_presence"
+output_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 
 # Crops the rectangle that contains a hand from the input image.
 node {
@@ -188,4 +189,5 @@ node {
   calculator: "HandGestureRecognitionCalculator"
   input_stream: "NORM_LANDMARKS:scaled_landmarks"
   input_stream: "NORM_RECT:hand_rect_for_next_frame"
+  output_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 }
diff --git a/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt
@@ -7,6 +7,7 @@ input_stream: "NORM_RECT:hand_rect"
 output_stream: "LANDMARKS:hand_landmarks"
 output_stream: "NORM_RECT:hand_rect_for_next_frame"
 output_stream: "PRESENCE:hand_presence"
+output_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 
 # Crops the rectangle that contains a hand from the input image.
 node {
@@ -183,4 +184,5 @@ node {
   calculator: "HandGestureRecognitionCalculator"
   input_stream: "NORM_LANDMARKS:scaled_landmarks"
   input_stream: "NORM_RECT:hand_rect_for_next_frame"
+  output_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 }
diff --git a/mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt
@@ -6,6 +6,7 @@ input_stream: "IMAGE:input_image"
 input_stream: "DETECTIONS:detections"
 input_stream: "LANDMARKS:landmarks"
 input_stream: "NORM_RECT:rect"
+input_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 output_stream: "IMAGE:output_image"
 
 # Converts detections to drawing primitives for annotation overlay.
@@ -98,5 +99,6 @@ node {
   input_stream: "detection_render_data"
   input_stream: "landmark_render_data"
   input_stream: "rect_render_data"
+  input_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
   output_stream: "OUTPUT_FRAME:output_image"
 }
diff --git a/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt
@@ -6,6 +6,7 @@ input_stream: "IMAGE:input_image"
 input_stream: "DETECTIONS:detections"
 input_stream: "LANDMARKS:landmarks"
 input_stream: "NORM_RECT:rect"
+input_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
 output_stream: "IMAGE:output_image"
 
 # Converts detections to drawing primitives for annotation overlay.
@@ -98,5 +99,6 @@ node {
   input_stream: "detection_render_data"
   input_stream: "landmark_render_data"
   input_stream: "rect_render_data"
+  input_stream: "RECOGNIZED_HAND_GESTURE:recognized_hand_gesture"
   output_stream: "OUTPUT_FRAME_GPU:output_image"
 }
diff --git a/mediapipe/util/annotation_renderer.cc b/mediapipe/util/annotation_renderer.cc
@@ -509,6 +509,20 @@ void AnnotationRenderer::DrawText(const RenderAnnotation& annotation) {
               /*bottomLeftOrigin=*/flip_text_vertically_);
 }
 
+void AnnotationRenderer::DrawText(std::string text)
+{
+  const int left = 275;
+  const int top = 50;
+  const cv::Point origin(left, top);
+  const int font_size = 35;
+  const int thickness = 5;
+  const cv::Scalar color = cv::Scalar(255.0, 0.0, 0.0);
+  const cv::HersheyFonts font_face = cv::FONT_HERSHEY_PLAIN;
+
+  const double font_scale = ComputeFontScale(font_face, font_size, thickness);
+  cv::putText(mat_image_, text, origin, font_face, font_scale, color, thickness);
+}
+
 double AnnotationRenderer::ComputeFontScale(int font_face, int font_size,
                                             int thickness) {
   double base_line;

diff --git a/mediapipe/util/annotation_renderer.h b/mediapipe/util/annotation_renderer.h
@@ -71,6 +71,8 @@ class AnnotationRenderer {
   // corner.
   void SetFlipTextVertically(bool flip);
 
+  void DrawText(std::string text);
+
  private:
   // Draws a rectangle on the image as described in the annotation.
   void DrawRectangle(const RenderAnnotation& annotation);