From 9ecdc998d19c2ae4145a1302f86f4047753919b6 Mon Sep 17 00:00:00 2001 From: Avi Romanoff Date: Mon, 20 Feb 2017 18:56:36 -0500 Subject: [PATCH] Run MtcnnDetector in separate thread --- src/FaceDetector.cpp | 20 +++++----- src/FaceDetector.hpp | 16 +++----- src/MtcnnDetector.cpp | 6 +++ src/MtcnnDetector.h | 9 +++-- src/main.cpp | 3 +- src/ofApp.cpp | 87 ++++++++++++++++++++++--------------------- src/ofApp.h | 7 ++-- 7 files changed, 78 insertions(+), 70 deletions(-) diff --git a/src/FaceDetector.cpp b/src/FaceDetector.cpp index d141476..655d826 100644 --- a/src/FaceDetector.cpp +++ b/src/FaceDetector.cpp @@ -7,26 +7,28 @@ // #include "FaceDetector.hpp" -#include "LandmarkCoreIncludes.h" -void FaceDetector::updateImage(cv::Mat grayscaleImage) { - grayscaleImage.copyTo(matGrayscale); +FaceDetector::FaceDetector() { + detector = new MtcnnDetector(); + isImageDirty = false; +} + +void FaceDetector::updateImage(cv::Mat newImage) { + ofLog() << "updateImage"; + newImage.copyTo(image); isImageDirty = true; } void FaceDetector::threadedFunction() { while(isThreadRunning()) { - - mutex.lock(); if (isImageDirty) { - vector confidences; - LandmarkDetector::DetectFacesHOG(faces_detected, matGrayscale, confidences); + mutex.lock(); + detectedFaces = detector->detectFaces(image); + mutex.unlock(); } - mutex.unlock(); } - // // Keep only non overlapping detections // NonOverlapingDetections(models, faces_detected); diff --git a/src/FaceDetector.hpp b/src/FaceDetector.hpp index de65cfa..4ca86f9 100644 --- a/src/FaceDetector.hpp +++ b/src/FaceDetector.hpp @@ -6,27 +6,23 @@ // // -#ifndef FaceDetector_hpp -#define FaceDetector_hpp - #include #include "ofThread.h" #include "ofxCv.h" - -#endif /* FaceDetector_hpp */ - +#include "MtcnnDetector.h" class FaceDetector : public ofThread { public: -// FaceDetector(); + FaceDetector(); void threadedFunction(); - void updateImage(cv::Mat grayscaleImage); + void updateImage(cv::Mat newImage); - vector> faces_detected; + mtcnn_detect_results detectedFaces; private: + MtcnnDetector *detector; bool isImageDirty; - cv::Mat_ matGrayscale; + cv::Mat image; }; diff --git a/src/MtcnnDetector.cpp b/src/MtcnnDetector.cpp index fbcb266..aa9a138 100644 --- a/src/MtcnnDetector.cpp +++ b/src/MtcnnDetector.cpp @@ -5,6 +5,7 @@ MtcnnDetector::MtcnnDetector() { Py_SetProgramName((char*)"mtcnn-bridge"); Py_Initialize(); + PyEval_InitThreads(); import_array(); PyObject *pModule, *pModuleName; @@ -30,6 +31,9 @@ MtcnnDetector::~MtcnnDetector() { } mtcnn_detect_results MtcnnDetector::detectFaces(const cv::Mat& mat) { + PyThreadState* state = PyEval_SaveThread(); + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); mtcnn_detect_results results; PyObject *pValue, *pArray, *pArgs; @@ -81,5 +85,7 @@ mtcnn_detect_results MtcnnDetector::detectFaces(const cv::Mat& mat) { results.pointGroups.push_back(points); } Py_XDECREF(pValue); + PyGILState_Release(gstate); + PyEval_RestoreThread(state); return results; } diff --git a/src/MtcnnDetector.h b/src/MtcnnDetector.h index 96f3645..34402e2 100644 --- a/src/MtcnnDetector.h +++ b/src/MtcnnDetector.h @@ -1,5 +1,6 @@ #include #include "ofxCv.h" +#include "ofThread.h" struct mtcnn_face_bbox { double x1; @@ -15,10 +16,12 @@ struct mtcnn_detect_results { }; class MtcnnDetector { - int width, height; - PyObject *pDetectFunc; - public: + +public: MtcnnDetector(); ~MtcnnDetector(); mtcnn_detect_results detectFaces(const cv::Mat& mat); + +private: + PyObject *pDetectFunc; }; diff --git a/src/main.cpp b/src/main.cpp index 21c7290..eabc1cb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3,7 +3,8 @@ //======================================================================== int main( ) { - ofSetupOpenGL(1920, 1080, OF_WINDOW); // <-------- setup the GL context + ofSetupOpenGL(640, 360, OF_WINDOW); + // ofSetupOpenGL(1920, 1080, OF_WINDOW); // Set the framework to something sane, otherwise the frame rate will be unbounded // and eat your CPU alive. diff --git a/src/ofApp.cpp b/src/ofApp.cpp index b1ad875..190a882 100644 --- a/src/ofApp.cpp +++ b/src/ofApp.cpp @@ -75,7 +75,7 @@ using namespace std; return rectVec; } - void bufferFrame::findFaces(MtcnnDetector *mtcnnDetector) { + void bufferFrame::findFaces(FaceDetector *faceDetector) { faces.clear(); //>>>>>>>> INSERT FACE DETECTOR >>>>>>>>>>>>>>> @@ -84,40 +84,46 @@ using namespace std; //vector randR = randNRects(numRandFaces); - + pRGB.resize(640, 360, OF_INTERPOLATE_NEAREST_NEIGHBOR); cv::Mat matAdjust = ofxCv::toCv(pRGB); - mtcnn_detect_results detectResults = mtcnnDetector->detectFaces(matAdjust); - - vector mxFaces = detectResults.bboxes; - int numFacesFound = mxFaces.size(); - - //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2GRAY); - //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2BGR); - - - //matDepth = cv::Mat(pixelsDepthRaw.getHeight(), pixelsDepthRaw.getWidth(), ofxCv::getCvImageType(pixelsDepthRaw), pixelsDepthRaw.getData(), 0); - - // here's how you use the mtcnn stuff - - // 1. load test image (for you, will be a frame) - // ofImage testImg; - // testImg.load(ofFilePath::getCurrentWorkingDirectory() + "/test.jpg"); - // cv::Mat imgMat = ofxCv::toCv(testImg); - // cv::Mat imgConv; - - // 2. convert to grayscale - // cv::cvtColor(imgMat, imgConv, CV_RGB2BGR); - - // 3. profit ! - // vector faces = mxnet_detect(imgConv); - // printf("%d faces found\n", faces.size()); - // for (mtcnn_face_bbox face : faces) { - // printf("\t[(%f,%f), (%f,%f), score = %f\n", face.x1, face.y1, face.x2`, face.y2, face.score); + // faceDetector->lock(); + // if (foobar) { + faceDetector->updateImage(matAdjust); + // foobar = false; // } - - - - + // mtcnn_detect_results detectResults = faceDetector->detectedFaces; + // // faceDetector->unlock(); + // + vector mxFaces = faceDetector->detectedFaces.bboxes; + int numFacesFound = mxFaces.size(); + // + // //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2GRAY); + // //cv::cvtColor(ofxCv::toCv(pRGB), matAdjust, CV_RGB2BGR); + // + // + // //matDepth = cv::Mat(pixelsDepthRaw.getHeight(), pixelsDepthRaw.getWidth(), ofxCv::getCvImageType(pixelsDepthRaw), pixelsDepthRaw.getData(), 0); + // + // // here's how you use the mtcnn stuff + // + // // 1. load test image (for you, will be a frame) + // // ofImage testImg; + // // testImg.load(ofFilePath::getCurrentWorkingDirectory() + "/test.jpg"); + // // cv::Mat imgMat = ofxCv::toCv(testImg); + // // cv::Mat imgConv; + // + // // 2. convert to grayscale + // // cv::cvtColor(imgMat, imgConv, CV_RGB2BGR); + //faceDetector->detectedFaces + // // 3. profit ! + // // vector faces = mxnet_detect(imgConv); + // // printf("%d faces found\n", faces.size()); + // // for (mtcnn_face_bbox face : faces) { + // // printf("\t[(%f,%f), (%f,%f), score = %f\n", face.x1, face.y1, face.x2`, face.y2, face.score); + // // } + // + // + // + // @@ -129,15 +135,11 @@ using namespace std; for(uint i = 0; i < numFacesFound; i++){ faceData face; - cout << "test 5" << endl; - face.r.x = mxFaces[i].x1; face.r.y = mxFaces[i].y1; face.r.width = abs(mxFaces[i].x2-mxFaces[i].x1); face.r.height = abs(mxFaces[i].y2-mxFaces[i].y1); - cout << "test 6" << endl; - faces.push_back(face); } @@ -161,15 +163,16 @@ using namespace std; std::exit(1); } - mtcnnDetector = new MtcnnDetector(); - - if(frame != NULL) frame = new bufferFrame(); + if(frame == NULL) frame = new bufferFrame(); frame->hasData = false; cout << "connected to kinect on port " << port << endl; freenect2 = kinect->freenect2; registration = kinect->registration; + + faceDetector = new FaceDetector(); + faceDetector->startThread(true); } void figKinect::update() { @@ -184,9 +187,7 @@ using namespace std; frame->rgbFrame = (libfreenect2::Frame *)kinect->getRgbFrame(); frame->depthFrame = (libfreenect2::Frame *)kinect->getDepthFrame(); - //cout << "1: " << ofGetElapsedTimeMillis() << endl; - frame->findFaces(mtcnnDetector); - //cout << "2: " << ofGetElapsedTimeMillis() << endl; + frame->findFaces(faceDetector); //frame->doRGBD(registration); diff --git a/src/ofApp.h b/src/ofApp.h index e468761..2e6da5f 100644 --- a/src/ofApp.h +++ b/src/ofApp.h @@ -4,7 +4,6 @@ #include "ofxCv.h" #include "ofxKinectV2.h" #include "LandmarkCoreIncludes.h" -#include "MtcnnDetector.h" #include #include "FaceDetector.hpp" @@ -48,7 +47,7 @@ class bufferFrame{ void draw(); - void findFaces(MtcnnDetector *mtcnnDetector); + void findFaces(FaceDetector *faceDetector); void doRGBD(libfreenect2::Registration* registration); @@ -73,8 +72,8 @@ class figKinect{ private: ofxKinectV2 *kinect = NULL; - bufferFrame* frame = NULL; - MtcnnDetector *mtcnnDetector = NULL; + bufferFrame *frame = NULL; + FaceDetector *faceDetector = NULL; };