Run MtcnnDetector in separate thread

src/FaceDetector.cpp

@@ -7,26 +7,28 @@
 //
 
 #include "FaceDetector.hpp"
-#include "LandmarkCoreIncludes.h"
 
-void FaceDetector::updateImage(cv::Mat grayscaleImage) {
-    grayscaleImage.copyTo(matGrayscale);
+FaceDetector::FaceDetector() {
+  detector = new MtcnnDetector();
+  isImageDirty = false;
+}
+
+void FaceDetector::updateImage(cv::Mat newImage) {
+    ofLog() << "updateImage";
+    newImage.copyTo(image);
     isImageDirty = true;
 }
 
 void FaceDetector::threadedFunction() {
 
     while(isThreadRunning()) {
-
-        mutex.lock();
         if (isImageDirty) {
-            vector<double> confidences;
-            LandmarkDetector::DetectFacesHOG(faces_detected, matGrayscale, confidences);
+            mutex.lock();
+            detectedFaces = detector->detectFaces(image);
+            mutex.unlock();
         }
-        mutex.unlock();
     }
 
-
 //    // Keep only non overlapping detections
 //    NonOverlapingDetections(models, faces_detected);
 

src/FaceDetector.hpp

@@ -6,27 +6,23 @@
 //
 //
 
-#ifndef FaceDetector_hpp
-#define FaceDetector_hpp
-
 #include <stdio.h>
 #include "ofThread.h"
 #include "ofxCv.h"
-
-#endif /* FaceDetector_hpp */
-
+#include "MtcnnDetector.h"
 
 class FaceDetector : public ofThread {
 
 public:
-//    FaceDetector();
+    FaceDetector();
     void threadedFunction();
-    void updateImage(cv::Mat grayscaleImage);
+    void updateImage(cv::Mat newImage);
 
-    vector<cv::Rect_<double>> faces_detected;
+    mtcnn_detect_results detectedFaces;
 
 private:
+    MtcnnDetector *detector;
     bool isImageDirty;
-    cv::Mat_<uchar> matGrayscale;
+    cv::Mat image;
 
 };

src/MtcnnDetector.cpp

@@ -5,6 +5,7 @@
 MtcnnDetector::MtcnnDetector() {
   Py_SetProgramName((char*)"mtcnn-bridge");
   Py_Initialize();
+  PyEval_InitThreads();
   import_array();
 
   PyObject *pModule, *pModuleName;
@@ -30,6 +31,9 @@ MtcnnDetector::~MtcnnDetector() {
 }
 
 mtcnn_detect_results MtcnnDetector::detectFaces(const cv::Mat& mat) {
+    PyThreadState* state = PyEval_SaveThread();
+    PyGILState_STATE gstate;
+    gstate = PyGILState_Ensure();
     mtcnn_detect_results results;
     PyObject *pValue, *pArray, *pArgs;
 
@@ -81,5 +85,7 @@ mtcnn_detect_results MtcnnDetector::detectFaces(const cv::Mat& mat) {
       results.pointGroups.push_back(points);
     }
     Py_XDECREF(pValue);
+    PyGILState_Release(gstate);
+    PyEval_RestoreThread(state);
     return results;
 }

src/MtcnnDetector.h

@@ -1,5 +1,6 @@
 #include <python2.7/Python.h>
 #include "ofxCv.h"
+#include "ofThread.h"
 
 struct mtcnn_face_bbox {
   double x1;
@@ -15,10 +16,12 @@ struct mtcnn_detect_results {
 };
 
 class MtcnnDetector {
-    int width, height;
-    PyObject *pDetectFunc;
-  public:
+
+public:
     MtcnnDetector();
    ~MtcnnDetector();
     mtcnn_detect_results detectFaces(const cv::Mat& mat);
+
+private:
+    PyObject *pDetectFunc;
 };

src/main.cpp

@@ -3,7 +3,8 @@
 
 //========================================================================
 int main( ) {
-	ofSetupOpenGL(1920, 1080, OF_WINDOW);			// <-------- setup the GL context
+	ofSetupOpenGL(640, 360, OF_WINDOW);
+	// ofSetupOpenGL(1920, 1080, OF_WINDOW);
 
     // Set the framework to something sane, otherwise the frame rate will be unbounded
     // and eat your CPU alive.

src/ofApp.cpp

@@ -75,7 +75,7 @@ using namespace std;
         return rectVec;
     }
 
-    void bufferFrame::findFaces(MtcnnDetector *mtcnnDetector) {
+    void bufferFrame::findFaces(FaceDetector *faceDetector) {
         faces.clear();
 
         //>>>>>>>> INSERT FACE DETECTOR >>>>>>>>>>>>>>>
@@ -84,40 +84,46 @@ using namespace std;
 
         //vector<rect> randR = randNRects(numRandFaces);
 
-
+        pRGB.resize(640, 360, OF_INTERPOLATE_NEAREST_NEIGHBOR);
         cv::Mat matAdjust = ofxCv::toCv(pRGB);
-        mtcnn_detect_results detectResults = mtcnnDetector->detectFaces(matAdjust);
-
-        vector<mtcnn_face_bbox> mxFaces = detectResults.bboxes;
-        int numFacesFound = mxFaces.size();
-
-        //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2GRAY);
-        //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2BGR);
-
-
-        //matDepth = cv::Mat(pixelsDepthRaw.getHeight(), pixelsDepthRaw.getWidth(), ofxCv::getCvImageType(pixelsDepthRaw), pixelsDepthRaw.getData(), 0);
-
-        // here's how you use the mtcnn stuff
-
-        // 1. load test image (for you, will be a frame)
-        // ofImage testImg;
-        // testImg.load(ofFilePath::getCurrentWorkingDirectory() + "/test.jpg");
-        // cv::Mat imgMat = ofxCv::toCv(testImg);
-        // cv::Mat imgConv;
-
-        // 2. convert to grayscale
-        // cv::cvtColor(imgMat, imgConv, CV_RGB2BGR);
-
-        // 3. profit !
-        // vector<mtcnn_face_bbox> faces = mxnet_detect(imgConv);
-        // printf("%d faces found\n", faces.size());
-        // for (mtcnn_face_bbox face : faces) {
-        //   printf("\t[(%f,%f), (%f,%f), score = %f\n", face.x1, face.y1, face.x2`, face.y2, face.score);
+        // faceDetector->lock();
+        // if (foobar) {
+          faceDetector->updateImage(matAdjust);
+          // foobar = false;
         // }
-
-
-
-
+        // mtcnn_detect_results detectResults = faceDetector->detectedFaces;
+        // // faceDetector->unlock();
+        //
+        vector<mtcnn_face_bbox> mxFaces = faceDetector->detectedFaces.bboxes;
+        int numFacesFound = mxFaces.size();
+        //
+        // //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2GRAY);
+        // //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2BGR);
+        //
+        //
+        // //matDepth = cv::Mat(pixelsDepthRaw.getHeight(), pixelsDepthRaw.getWidth(), ofxCv::getCvImageType(pixelsDepthRaw), pixelsDepthRaw.getData(), 0);
+        //
+        // // here's how you use the mtcnn stuff
+        //
+        // // 1. load test image (for you, will be a frame)
+        // // ofImage testImg;
+        // // testImg.load(ofFilePath::getCurrentWorkingDirectory() + "/test.jpg");
+        // // cv::Mat imgMat = ofxCv::toCv(testImg);
+        // // cv::Mat imgConv;
+        //
+        // // 2. convert to grayscale
+        // // cv::cvtColor(imgMat, imgConv, CV_RGB2BGR);
+        //faceDetector->detectedFaces
+        // // 3. profit !
+        // // vector<mtcnn_face_bbox> faces = mxnet_detect(imgConv);
+        // // printf("%d faces found\n", faces.size());
+        // // for (mtcnn_face_bbox face : faces) {
+        // //   printf("\t[(%f,%f), (%f,%f), score = %f\n", face.x1, face.y1, face.x2`, face.y2, face.score);
+        // // }
+        //
+        //
+        //
+        //
 
 
 
@@ -129,15 +135,11 @@ using namespace std;
         for(uint i = 0; i < numFacesFound; i++){
             faceData face;
 
-            cout << "test 5" << endl;
-
             face.r.x = mxFaces[i].x1;
             face.r.y = mxFaces[i].y1;
             face.r.width = abs(mxFaces[i].x2-mxFaces[i].x1);
             face.r.height = abs(mxFaces[i].y2-mxFaces[i].y1);
 
-            cout << "test 6" << endl;
-
             faces.push_back(face);
         }
 
@@ -161,15 +163,16 @@ using namespace std;
             std::exit(1);
         }
 
-        mtcnnDetector = new MtcnnDetector();
-
-        if(frame != NULL) frame = new bufferFrame();
+        if(frame == NULL) frame = new bufferFrame();
         frame->hasData = false;
 
         cout << "connected to kinect on port " << port << endl;
 
         freenect2 = kinect->freenect2;
         registration = kinect->registration;
+
+        faceDetector = new FaceDetector();
+        faceDetector->startThread(true);
     }
 
     void figKinect::update() {
@@ -184,9 +187,7 @@ using namespace std;
             frame->rgbFrame = (libfreenect2::Frame *)kinect->getRgbFrame();
             frame->depthFrame = (libfreenect2::Frame *)kinect->getDepthFrame();
 
-	        //cout << "1: " << ofGetElapsedTimeMillis() << endl;
-            frame->findFaces(mtcnnDetector);
-	        //cout << "2: " << ofGetElapsedTimeMillis() << endl;
+            frame->findFaces(faceDetector);
 
             //frame->doRGBD(registration);
 

src/ofApp.h

@@ -4,7 +4,6 @@
 #include "ofxCv.h"
 #include "ofxKinectV2.h"
 #include "LandmarkCoreIncludes.h"
-#include "MtcnnDetector.h"
 #include <FaceAnalyser.h>
 #include "FaceDetector.hpp"
 
@@ -48,7 +47,7 @@ class bufferFrame{
 
     void draw();
 
-    void findFaces(MtcnnDetector *mtcnnDetector);
+    void findFaces(FaceDetector *faceDetector);
 
     void doRGBD(libfreenect2::Registration* registration);
 
@@ -73,8 +72,8 @@ class figKinect{
 
 private:
     ofxKinectV2 *kinect = NULL;
-    bufferFrame* frame = NULL;
-    MtcnnDetector *mtcnnDetector = NULL;
+    bufferFrame *frame = NULL;
+    FaceDetector *faceDetector = NULL;
 };