Face/hand working with any output resolution

gineshidalgo99 · gineshidalgo99 · commit 29ea7e24dce4 · 2018-04-01T19:49:19.000-04:00
diff --git a/doc/release_notes.md b/doc/release_notes.md
@@ -215,7 +215,9 @@ OpenPose Library - Release Notes
 1. Main improvements:
     1. Flir cameras: Added software trigger and a dedicated thread to keep reading images to remove latency (analogously to webcamReader).
 2. Functions or parameters renamed:
+    1. Removed scale parameter from hand and face rectangle extractor (causing wrong results if custom `--output_resolution`).
 3. Main bugs fixed:
+    1. Hand and face work properly again with any `--output_resolution`.
 
 
 
diff --git a/include/openpose/face/faceDetector.hpp b/include/openpose/face/faceDetector.hpp
@@ -11,7 +11,7 @@ namespace op
     public:
         explicit FaceDetector(const PoseModel poseModel);
 
-        std::vector<Rectangle<float>> detectFaces(const Array<float>& poseKeypoints, const double scaleInputToOutput) const;
+        std::vector<Rectangle<float>> detectFaces(const Array<float>& poseKeypoints) const;
 
     private:
         const unsigned int mNeck;
diff --git a/include/openpose/face/wFaceDetector.hpp b/include/openpose/face/wFaceDetector.hpp
@@ -56,7 +56,7 @@ namespace op
                 const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__);
                 // Detect people face
                 for (auto& tDatum : *tDatums)
-                    tDatum.faceRectangles = spFaceDetector->detectFaces(tDatum.poseKeypoints, tDatum.scaleInputToOutput);
+                    tDatum.faceRectangles = spFaceDetector->detectFaces(tDatum.poseKeypoints);
                 // Profiling speed
                 Profiler::timerEnd(profilerKey);
                 Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__);
diff --git a/include/openpose/hand/handDetector.hpp b/include/openpose/hand/handDetector.hpp
@@ -14,9 +14,9 @@ namespace op
     public:
         explicit HandDetector(const PoseModel poseModel);
 
-        std::vector<std::array<Rectangle<float>, 2>> detectHands(const Array<float>& poseKeypoints, const double scaleInputToOutput) const;
+        std::vector<std::array<Rectangle<float>, 2>> detectHands(const Array<float>& poseKeypoints) const;
 
-        std::vector<std::array<Rectangle<float>, 2>> trackHands(const Array<float>& poseKeypoints, const double scaleInputToOutput);
+        std::vector<std::array<Rectangle<float>, 2>> trackHands(const Array<float>& poseKeypoints);
 
         void updateTracker(const std::array<Array<float>, 2>& handKeypoints, const unsigned long long id);
 
diff --git a/include/openpose/hand/wHandDetector.hpp b/include/openpose/hand/wHandDetector.hpp
@@ -56,7 +56,7 @@ namespace op
                 const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__);
                 // Detect people hand
                 for (auto& tDatum : *tDatums)
-                    tDatum.handRectangles = spHandDetector->detectHands(tDatum.poseKeypoints, tDatum.scaleInputToOutput);
+                    tDatum.handRectangles = spHandDetector->detectHands(tDatum.poseKeypoints);
                 // Profiling speed
                 Profiler::timerEnd(profilerKey);
                 Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__);
diff --git a/include/openpose/hand/wHandDetectorTracking.hpp b/include/openpose/hand/wHandDetectorTracking.hpp
@@ -56,7 +56,7 @@ namespace op
                 const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__);
                 // Detect people hand
                 for (auto& tDatum : *tDatums)
-                    tDatum.handRectangles = spHandDetector->trackHands(tDatum.poseKeypoints, tDatum.scaleInputToOutput);
+                    tDatum.handRectangles = spHandDetector->trackHands(tDatum.poseKeypoints);
                 // Profiling speed
                 Profiler::timerEnd(profilerKey);
                 Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__);
diff --git a/src/openpose/face/faceDetector.cpp b/src/openpose/face/faceDetector.cpp
@@ -15,9 +15,11 @@ namespace op
     {
     }
 
-    inline Rectangle<float> getFaceFromPoseKeypoints(const Array<float>& poseKeypoints, const unsigned int personIndex, const unsigned int neck,
-                                                     const unsigned int headNose, const unsigned int lEar, const unsigned int rEar,
-                                                     const unsigned int lEye, const unsigned int rEye, const float threshold)
+    inline Rectangle<float> getFaceFromPoseKeypoints(const Array<float>& poseKeypoints, const unsigned int personIndex,
+                                                     const unsigned int neck, const unsigned int headNose,
+                                                     const unsigned int lEar, const unsigned int rEar,
+                                                     const unsigned int lEye, const unsigned int rEye,
+                                                     const float threshold)
     {
         try
         {
@@ -58,13 +60,17 @@ namespace op
                         {
                             pointTopLeft.x += (posePtr[lEye*3] + posePtr[lEar*3] + posePtr[headNose*3]) / 3.f;
                             pointTopLeft.y += (posePtr[lEye*3+1] + posePtr[lEar*3+1] + posePtr[headNose*3+1]) / 3.f;
-                            faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, lEye) + getDistance(poseKeypoints, personIndex, headNose, lEar) + getDistance(poseKeypoints, personIndex, neck, headNose));
+                            faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, lEye)
+                                                 + getDistance(poseKeypoints, personIndex, headNose, lEar)
+                                                 + getDistance(poseKeypoints, personIndex, neck, headNose));
                         }
                         else // if(lEyeScoreAbove)
                         {
                             pointTopLeft.x += (posePtr[rEye*3] + posePtr[rEar*3] + posePtr[headNose*3]) / 3.f;
                             pointTopLeft.y += (posePtr[rEye*3+1] + posePtr[rEar*3+1] + posePtr[headNose*3+1]) / 3.f;
-                            faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, rEye) + getDistance(poseKeypoints, personIndex, headNose, rEar) + getDistance(poseKeypoints, personIndex, neck, headNose));
+                            faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, rEye)
+                                                 + getDistance(poseKeypoints, personIndex, headNose, rEar)
+                                                 + getDistance(poseKeypoints, personIndex, neck, headNose));
                         }
                     }
                     // else --> 2 * dist(neck, headNose)
@@ -108,7 +114,7 @@ namespace op
         }
     }
 
-    std::vector<Rectangle<float>> FaceDetector::detectFaces(const Array<float>& poseKeypoints, const double scaleInputToOutput) const
+    std::vector<Rectangle<float>> FaceDetector::detectFaces(const Array<float>& poseKeypoints) const
     {
         try
         {
@@ -119,7 +125,8 @@ namespace op
             // Otherwise, get face position(s)
             if (!poseKeypoints.empty())
                 for (auto person = 0 ; person < numberPeople ; person++)
-                    faceRectangles.at(person) = getFaceFromPoseKeypoints(poseKeypoints, person, mNeck, mNose, mLEar, mREar, mLEye, mREye, threshold) / (float)scaleInputToOutput;
+                    faceRectangles.at(person) = getFaceFromPoseKeypoints(
+                        poseKeypoints, person, mNeck, mNose, mLEar, mREar, mLEye, mREye, threshold);
             return faceRectangles;
         }
         catch (const std::exception& e)
diff --git a/src/openpose/hand/handDetector.cpp b/src/openpose/hand/handDetector.cpp
@@ -130,7 +130,7 @@ namespace op
     {
     }
 
-    std::vector<std::array<Rectangle<float>, 2>> HandDetector::detectHands(const Array<float>& poseKeypoints, const double scaleInputToOutput) const
+    std::vector<std::array<Rectangle<float>, 2>> HandDetector::detectHands(const Array<float>& poseKeypoints) const
     {
         try
         {
@@ -148,8 +148,6 @@ namespace op
                         mPoseIndexes[(int)PosePart::LShoulder], mPoseIndexes[(int)PosePart::RWrist],
                         mPoseIndexes[(int)PosePart::RElbow], mPoseIndexes[(int)PosePart::RShoulder], threshold
                     );
-                    handRectangles.at(person).at(0) /= (float) scaleInputToOutput;
-                    handRectangles.at(person).at(1) /= (float) scaleInputToOutput;
                 }
             }
             return handRectangles;
@@ -161,13 +159,13 @@ namespace op
         }
     }
 
-    std::vector<std::array<Rectangle<float>, 2>> HandDetector::trackHands(const Array<float>& poseKeypoints, const double scaleInputToOutput)
+    std::vector<std::array<Rectangle<float>, 2>> HandDetector::trackHands(const Array<float>& poseKeypoints)
     {
         try
         {
             std::lock_guard<std::mutex> lock{mMutex};
             // Baseline detectHands
-            auto handRectangles = detectHands(poseKeypoints, scaleInputToOutput);
+            auto handRectangles = detectHands(poseKeypoints);
             // If previous hands saved
             for (auto& handRectangle : handRectangles)
             {
diff --git a/src/openpose/producer/spinnakerWrapper.cpp b/src/openpose/producer/spinnakerWrapper.cpp
@@ -354,8 +354,10 @@ namespace op
                             }
                             if (imagesExtracted)
                             {
-                                const std::lock_guard<std::mutex> lock{mBufferMutex};
+                                std::unique_lock<std::mutex> lock{mBufferMutex};
                                 std::swap(mBuffer, imagePtrs);
+                                lock.unlock();
+                                std::this_thread::sleep_for(std::chrono::microseconds{1});
                             }
                         }
                     }
@@ -408,23 +410,9 @@ namespace op
                             std::this_thread::sleep_for(std::chrono::microseconds{5});
                         }
                     }
-                    // Commented code was supposed to clean buffer, but `NewestFirstOverwrite` does that
                     // Getting frames
                     // Retrieve next received image and ensure image completion
                     // Spinnaker::ImagePtr imagePtr = cameraPtrs.at(i)->GetNextImage();
-                    // Clean buffer + retrieve next received image + ensure image completion
-                    // auto durationMs = 0.;
-                    // // for (auto counter = 0 ; counter < 10 ; counter++)
-                    // while (durationMs < 1.)
-                    // {
-                    //     const auto begin = std::chrono::high_resolution_clock::now();
-                    //     for (auto i = 0u; i < cameraPtrs.size(); i++)
-                    //         imagePtrs.at(i) = cameraPtrs.at(i)->GetNextImage();
-                    //     durationMs = std::chrono::duration_cast<std::chrono::nanoseconds>(
-                    //         std::chrono::high_resolution_clock::now()-begin
-                    //     ).count() * 1e-6;
-                    //     // log("Time extraction (ms): " + std::to_string(durationMs), Priority::High);
-                    // }
 
                     // All images completed
                     bool imagesExtracted = true;

Original file line number	Diff line number	Diff line change
`@@ -15,9 +15,11 @@ namespace op`
`15`	`15`	`{`
`16`	`16`	`}`
`17`	`17`
`18`		`- inline Rectangle<float> getFaceFromPoseKeypoints(const Array<float>& poseKeypoints, const unsigned int personIndex, const unsigned int neck,`
`19`		`- const unsigned int headNose, const unsigned int lEar, const unsigned int rEar,`
`20`		`- const unsigned int lEye, const unsigned int rEye, const float threshold)`
	`18`	`+ inline Rectangle<float> getFaceFromPoseKeypoints(const Array<float>& poseKeypoints, const unsigned int personIndex,`
	`19`	`+ const unsigned int neck, const unsigned int headNose,`
	`20`	`+ const unsigned int lEar, const unsigned int rEar,`
	`21`	`+ const unsigned int lEye, const unsigned int rEye,`
	`22`	`+ const float threshold)`
`21`	`23`	`{`
`22`	`24`	`try`
`23`	`25`	`{`
`@@ -58,13 +60,17 @@ namespace op`
`58`	`60`	`{`
`59`	`61`	`pointTopLeft.x += (posePtr[lEye3] + posePtr[lEar3] + posePtr[headNose*3]) / 3.f;`
`60`	`62`	`pointTopLeft.y += (posePtr[lEye3+1] + posePtr[lEar3+1] + posePtr[headNose*3+1]) / 3.f;`
`61`		`- faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, lEye) + getDistance(poseKeypoints, personIndex, headNose, lEar) + getDistance(poseKeypoints, personIndex, neck, headNose));`
	`63`	`+ faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, lEye)`
	`64`	`+ + getDistance(poseKeypoints, personIndex, headNose, lEar)`
	`65`	`+ + getDistance(poseKeypoints, personIndex, neck, headNose));`
`62`	`66`	`}`
`63`	`67`	`else // if(lEyeScoreAbove)`
`64`	`68`	`{`
`65`	`69`	`pointTopLeft.x += (posePtr[rEye3] + posePtr[rEar3] + posePtr[headNose*3]) / 3.f;`
`66`	`70`	`pointTopLeft.y += (posePtr[rEye3+1] + posePtr[rEar3+1] + posePtr[headNose*3+1]) / 3.f;`
`67`		`- faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, rEye) + getDistance(poseKeypoints, personIndex, headNose, rEar) + getDistance(poseKeypoints, personIndex, neck, headNose));`
	`71`	`+ faceSize += 0.85f * (getDistance(poseKeypoints, personIndex, headNose, rEye)`
	`72`	`+ + getDistance(poseKeypoints, personIndex, headNose, rEar)`
	`73`	`+ + getDistance(poseKeypoints, personIndex, neck, headNose));`
`68`	`74`	`}`
`69`	`75`	`}`
`70`	`76`	`// else --> 2 * dist(neck, headNose)`
`@@ -108,7 +114,7 @@ namespace op`
`108`	`114`	`}`
`109`	`115`	`}`
`110`	`116`
`111`		`- std::vector<Rectangle<float>> FaceDetector::detectFaces(const Array<float>& poseKeypoints, const double scaleInputToOutput) const`
	`117`	`+ std::vector<Rectangle<float>> FaceDetector::detectFaces(const Array<float>& poseKeypoints) const`
`112`	`118`	`{`
`113`	`119`	`try`
`114`	`120`	`{`
`@@ -119,7 +125,8 @@ namespace op`
`119`	`125`	`// Otherwise, get face position(s)`
`120`	`126`	`if (!poseKeypoints.empty())`
`121`	`127`	`for (auto person = 0 ; person < numberPeople ; person++)`
`122`		`- faceRectangles.at(person) = getFaceFromPoseKeypoints(poseKeypoints, person, mNeck, mNose, mLEar, mREar, mLEye, mREye, threshold) / (float)scaleInputToOutput;`
	`128`	`+ faceRectangles.at(person) = getFaceFromPoseKeypoints(`
	`129`	`+ poseKeypoints, person, mNeck, mNose, mLEar, mREar, mLEye, mREye, threshold);`
`123`	`130`	`return faceRectangles;`
`124`	`131`	`}`
`125`	`132`	`catch (const std::exception& e)`
Original file line number	Diff line number	Diff line change
`@@ -130,7 +130,7 @@ namespace op`
`130`	`130`	`{`
`131`	`131`	`}`
`132`	`132`
`133`		`- std::vector<std::array<Rectangle<float>, 2>> HandDetector::detectHands(const Array<float>& poseKeypoints, const double scaleInputToOutput) const`
	`133`	`+ std::vector<std::array<Rectangle<float>, 2>> HandDetector::detectHands(const Array<float>& poseKeypoints) const`
`134`	`134`	`{`
`135`	`135`	`try`
`136`	`136`	`{`
`@@ -148,8 +148,6 @@ namespace op`
`148`	`148`	`mPoseIndexes[(int)PosePart::LShoulder], mPoseIndexes[(int)PosePart::RWrist],`
`149`	`149`	`mPoseIndexes[(int)PosePart::RElbow], mPoseIndexes[(int)PosePart::RShoulder], threshold`
`150`	`150`	`);`
`151`		`- handRectangles.at(person).at(0) /= (float) scaleInputToOutput;`
`152`		`- handRectangles.at(person).at(1) /= (float) scaleInputToOutput;`
`153`	`151`	`}`
`154`	`152`	`}`
`155`	`153`	`return handRectangles;`
`@@ -161,13 +159,13 @@ namespace op`
`161`	`159`	`}`
`162`	`160`	`}`
`163`	`161`
`164`		`- std::vector<std::array<Rectangle<float>, 2>> HandDetector::trackHands(const Array<float>& poseKeypoints, const double scaleInputToOutput)`
	`162`	`+ std::vector<std::array<Rectangle<float>, 2>> HandDetector::trackHands(const Array<float>& poseKeypoints)`
`165`	`163`	`{`
`166`	`164`	`try`
`167`	`165`	`{`
`168`	`166`	`std::lock_guard<std::mutex> lock{mMutex};`
`169`	`167`	`// Baseline detectHands`
`170`		`- auto handRectangles = detectHands(poseKeypoints, scaleInputToOutput);`
	`168`	`+ auto handRectangles = detectHands(poseKeypoints);`
`171`	`169`	`// If previous hands saved`
`172`	`170`	`for (auto& handRectangle : handRectangles)`
`173`	`171`	`{`
Original file line number	Diff line number	Diff line change
`@@ -354,8 +354,10 @@ namespace op`
`354`	`354`	`}`
`355`	`355`	`if (imagesExtracted)`
`356`	`356`	`{`
`357`		`- const std::lock_guard<std::mutex> lock{mBufferMutex};`
	`357`	`+ std::unique_lock<std::mutex> lock{mBufferMutex};`
`358`	`358`	`std::swap(mBuffer, imagePtrs);`
	`359`	`+ lock.unlock();`
	`360`	`+ std::this_thread::sleep_for(std::chrono::microseconds{1});`
`359`	`361`	`}`
`360`	`362`	`}`
`361`	`363`	`}`
`@@ -408,23 +410,9 @@ namespace op`
`408`	`410`	`std::this_thread::sleep_for(std::chrono::microseconds{5});`
`409`	`411`	`}`
`410`	`412`	`}`
`411`		- // Commented code was supposed to clean buffer, but `NewestFirstOverwrite` does that
`412`	`413`	`// Getting frames`
`413`	`414`	`// Retrieve next received image and ensure image completion`
`414`	`415`	`// Spinnaker::ImagePtr imagePtr = cameraPtrs.at(i)->GetNextImage();`
`415`		`- // Clean buffer + retrieve next received image + ensure image completion`
`416`		`- // auto durationMs = 0.;`
`417`		`- // // for (auto counter = 0 ; counter < 10 ; counter++)`
`418`		`- // while (durationMs < 1.)`
`419`		`- // {`
`420`		`- // const auto begin = std::chrono::high_resolution_clock::now();`
`421`		`- // for (auto i = 0u; i < cameraPtrs.size(); i++)`
`422`		`- // imagePtrs.at(i) = cameraPtrs.at(i)->GetNextImage();`
`423`		`- // durationMs = std::chrono::duration_cast<std::chrono::nanoseconds>(`
`424`		`- // std::chrono::high_resolution_clock::now()-begin`
`425`		`- // ).count() * 1e-6;`
`426`		`- // // log("Time extraction (ms): " + std::to_string(durationMs), Priority::High);`
`427`		`- // }`
`428`	`416`
`429`	`417`	`// All images completed`
`430`	`418`	`bool imagesExtracted = true;`