Changed bounding box draw in Object Detector effect

include/CVStabilization.h

@@ -96,7 +96,7 @@ class CVStabilization {
 	ProcessingController *processingController;
 
     // Track current frame features and find the relative transformation
-    void TrackFrameFeatures(cv::Mat frame, size_t frameNum);
+    bool TrackFrameFeatures(cv::Mat frame, size_t frameNum);
 
     std::vector<CamTrajectory> ComputeFramesTrajectory();
     std::map<size_t,CamTrajectory> SmoothTrajectory(std::vector <CamTrajectory> &trajectory);

include/effects/ObjectDetection.h

@@ -36,6 +36,7 @@
 #include <cmath>
 #include <stdio.h>
 #include <memory>
+#include <opencv2/opencv.hpp>
 #include "../Color.h"
 #include "../Json.h"
 #include "../KeyFrame.h"
@@ -70,6 +71,8 @@ namespace openshot
 		std::string protobuf_data_path;
 		std::map<size_t, DetectionData> detectionsData;
 		std::vector<std::string> classNames;
+
+		std::vector<cv::Scalar> classesColor;
 
 		/// Init effect settings
 		void init_effect_details();

src/CVObjectDetection.cpp

@@ -166,17 +166,21 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
     std::vector<int> indices;
     cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
 
-    // std::vector<cv::Rect> sortBoxes;
-    // for(auto box : boxes)
-    //     sortBoxes.push_back(box);
-    // sort.update(sortBoxes, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2)));
+    std::vector<cv::Rect> sortBoxes;
+    for(auto box : boxes)
+        sortBoxes.push_back(box);
+    sort.update(sortBoxes, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2)), confidences, classIds);
 
 
-    // sortBoxes.clear();
-    // for(auto TBox : sort.frameTrackingResult)
-    //     if(TBox.frame == frameId){
-    //         sortBoxes.push_back(TBox.box);
-    //     }
+    sortBoxes.clear(); boxes.clear(); confidences.clear(); classIds.clear();
+
+    for(auto TBox : sort.frameTrackingResult){
+        if(TBox.frame == frameId){
+            boxes.push_back(TBox.box);
+            confidences.push_back(TBox.confidence);
+            classIds.push_back(TBox.classId);
+        }
+    }
 
     // for(int i = 0; i<boxes.size(); i++){
     //     bool found = false;

src/CVStabilization.cpp

@@ -35,6 +35,8 @@ CVStabilization::CVStabilization(std::string processInfoJson, ProcessingControll
 : processingController(&processingController){
     SetJson(processInfoJson);
 }
+double mediax=0, mediay=0, mediaa=0, mediastatus=0, maiora = 0, maiorx = 0, maiory = 0;
+int maiorstatus=0;
 
 // Process clip and store necessary stabilization data
 void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
@@ -64,7 +66,8 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t
         cv::Mat cvimage = f->GetImageCV();
         cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
 
-        TrackFrameFeatures(cvimage, frame_number);
+        if(! TrackFrameFeatures(cvimage, frame_number))
+            prev_to_cur_transform.push_back(TransformParam(0, 0, 0));
 
         // Update progress
         processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
@@ -81,17 +84,17 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t
 }
 
 // Track current frame features and find the relative transformation
-void CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
+bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
     std::cout<<"frame "<<frameNum<<"\n";
     if(cv::countNonZero(frame) < 1){
-        last_T = cv::Mat();
-        prev_grey = cv::Mat();
-        return;
+        // last_T = cv::Mat();
+        // prev_grey = cv::Mat();
+        return false;
     }
 
     if(prev_grey.empty()){
         prev_grey = frame;
-        return;
+        return false;
     }
 
     // OpticalFlow features vector
@@ -100,10 +103,14 @@ void CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
     std::vector <uchar> status;
     std::vector <float> err;
     // Extract new image features
-    cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30);
+    cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 15);
     // Track features
     cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err);
     // Remove untracked features
+    mediastatus+=status.size();
+    if(status.size() > maiorstatus)
+        maiorstatus = status.size();
+
     for(size_t i=0; i < status.size(); i++) {
         if(status[i]) {
             prev_corner2.push_back(prev_corner[i]);
@@ -114,18 +121,15 @@ void CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
     if(prev_corner2.empty() || cur_corner2.empty()){
         last_T = cv::Mat();
         prev_grey = cv::Mat();
-        return;
+        return false;
     }
 
     // Translation + rotation only
-    cv::Mat T = estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing
+    cv::Mat T = cv::estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing
 
     double da, dx, dy;
     if(T.size().width == 0 || T.size().height == 0){
-
-        dx = 0;
-        dy = 0;
-        da = 0;
+        return false;
     }
     else{
         // If no transformation is found, just use the last known good transform.
@@ -137,10 +141,26 @@ void CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
         da = atan2(T.at<double>(1,0), T.at<double>(0,0));
     }
 
+    if(dx > 100 || dy > 100 || da > 0.1){
+        return false;
+    }
+
+    mediax+=fabs(dx);
+    mediay+=fabs(dy);
+    mediaa+=fabs(da);
+
+    if(fabs(dx) > maiorx)
+        maiorx = dx;
+    if(fabs(dy) > maiory)
+        maiory = dy;
+    if(fabs(da) > maiora)
+        maiora = da;
+
+    std::cout<<dx<<" "<<dy<<" "<<da<<"\n";
+
     T.copyTo(last_T);
 
     prev_to_cur_transform.push_back(TransformParam(dx, dy, da));
-    std::cout<<"10\n";
     frame.copyTo(prev_grey);
 
     // Show processing info

src/CVTracker.cpp

@@ -154,8 +154,10 @@ bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
         float fh = frame.size().height;
 
         std::vector<cv::Rect> bboxes = {bbox};
+        std::vector<float> confidence = {1.0};
+        std::vector<int> classId = {1};
 
-        sort.update(bboxes, frameId, sqrt(pow(frame.rows, 2) + pow(frame.cols, 2)));
+        sort.update(bboxes, frameId, sqrt(pow(frame.rows, 2) + pow(frame.cols, 2)), confidence, classId);
 
         for(auto TBox : sort.frameTrackingResult)
             bbox = TBox.box;

src/effects/ObjectDetection.cpp

@@ -133,8 +133,9 @@ std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, i
 
 void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame)
 {
+
     //Draw a rectangle displaying the bounding box
-    cv::rectangle(frame, box, cv::Scalar(255, 178, 50), 3);
+    cv::rectangle(frame, box, classesColor[classId], 2);
 
     //Get the label for the class name and its confidence
     std::string label = cv::format("%.2f", conf);
@@ -144,12 +145,15 @@ void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat&
         label = classNames[classId] + ":" + label;
     }
 
-    //Display the label at the box.y of the bounding box
+    //Display the label at the top of the bounding box
     int baseLine;
-    cv::Size labelSize = getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
-    box.y = std::max((int)box.y, labelSize.height);
-    cv::rectangle(frame, cv::Point(box.x, box.y - round(1.5*labelSize.height)), cv::Point(box.x + round(1.5*labelSize.width), box.y + baseLine), cv::Scalar(255, 255, 255), cv::FILLED);
-    putText(frame, label, cv::Point(box.x, box.y), cv::FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0,0,0),1);
+    cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
+
+    double left = box.x;
+    double top = std::max((int)box.y, labelSize.height);
+
+    cv::rectangle(frame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine), classesColor[classId], cv::FILLED);
+    putText(frame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
 }
 
 // Load protobuf data file
@@ -172,6 +176,7 @@ bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){
 
     for(int i = 0; i < objMessage.classnames_size(); i++){
         classNames.push_back(objMessage.classnames(i));
+        classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
     }
 
     // Iterate over all frames of the saved message

src/sort_filter/KalmanTracker.h

@@ -24,7 +24,7 @@ class KalmanTracker
 		m_id = kf_count;
 		//kf_count++;
 	}
-	KalmanTracker(StateType initRect)
+	KalmanTracker(StateType initRect, float confidence, int classId) : confidence(confidence), classId(classId)
 	{
 		init_kf(initRect);
 		m_time_since_update = 0;
@@ -54,6 +54,8 @@ class KalmanTracker
 	int m_hit_streak;
 	int m_age;
 	int m_id;
+	float confidence;
+	int classId;
 
 private:
 	void init_kf(StateType stateMat);

src/sort_filter/sort.cpp

@@ -37,7 +37,7 @@ double SortTracker::GetCentroidsDistance(
 	return distance;
 }
 
-void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double image_diagonal)
+void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double image_diagonal, std::vector<float> confidences, std::vector<int> classIds)
 {
 	vector<TrackingBox> detections;
 	if (trackers.size() == 0) // the first frame met
@@ -49,9 +49,11 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 			TrackingBox tb;
 
 			tb.box = cv::Rect_<float>(detections_cv[i]);
+			tb.classId = classIds[i];
+			tb.confidence = confidences[i];
 			detections.push_back(tb);
 
-			KalmanTracker trk = KalmanTracker(detections[i].box);
+			KalmanTracker trk = KalmanTracker(detections[i].box, detections[i].confidence, detections[i].classId);
 			trackers.push_back(trk);
 		}
 		return;
@@ -62,6 +64,8 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 		{
 			TrackingBox tb;
 			tb.box = cv::Rect_<float>(detections_cv[i]);
+			tb.classId = classIds[i];
+			tb.confidence = confidences[i];
 			detections.push_back(tb);
 		}
 		for (auto it = frameTrackingResult.begin(); it != frameTrackingResult.end(); it++)
@@ -156,12 +160,14 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 		trkIdx = matchedPairs[i].x;
 		detIdx = matchedPairs[i].y;
 		trackers[trkIdx].update(detections[detIdx].box);
+		trackers[trkIdx].classId = detections[detIdx].classId;
+		trackers[trkIdx].confidence = detections[detIdx].confidence;
 	}
 
 	// create and initialise new trackers for unmatched detections
 	for (auto umd : unmatchedDetections)
 	{
-		KalmanTracker tracker = KalmanTracker(detections[umd].box);
+		KalmanTracker tracker = KalmanTracker(detections[umd].box, detections[umd].confidence, detections[umd].classId);
 		trackers.push_back(tracker);
 	}
 
@@ -189,6 +195,8 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 			res.box = trackers[i].get_state();
 			res.id = trackers[i].m_id;
 			res.frame = frame_count;
+			res.classId = trackers[i].classId;
+			res.confidence = trackers[i].confidence;
 			frameTrackingResult.push_back(res);
 		}
 

src/sort_filter/sort.hpp

@@ -32,7 +32,7 @@ class SortTracker
 	// Initialize tracker
 
 	// Update position based on the new frame
-	void update(std::vector<cv::Rect> detection, int frame_count, double image_diagonal);
+	void update(std::vector<cv::Rect> detection, int frame_count, double image_diagonal, std::vector<float> confidences, std::vector<int> classIds);
 	double GetIOU(cv::Rect_<float> bb_test, cv::Rect_<float> bb_gt);
 	double GetCentroidsDistance(cv::Rect_<float> bb_test, cv::Rect_<float> bb_gt);
 	std::vector<KalmanTracker> trackers;