Merge pull request #649 from OpenShot/effect-parenting

Effect parenting

bindings/python/openshot.i

@@ -120,6 +120,7 @@
 		#include "effects/Stabilizer.h"
 		#include "effects/Tracker.h"
 		#include "effects/ObjectDetection.h"
+		#include "TrackedObjectBBox.h"
 	%}
 #endif
 
@@ -291,6 +292,7 @@
 
 #ifdef USE_OPENCV
 	%include "ClipProcessingJobs.h"
+	%include "TrackedObjectBBox.h"
 #endif
 
 #ifdef USE_IMAGEMAGICK

examples/CMakeLists.txt

@@ -63,3 +63,15 @@ if (BLACKMAGIC_FOUND)
 	# Link test executable to the new library
 	target_link_libraries(openshot-blackmagic openshot)
 endif()
+
+############### OPENCV EXAMPLE ################
+#if (DEFINED CACHE{HAVE_OPENCV})
+#	# Create test executable
+#	add_executable(openshot-example-opencv
+#			Example_opencv.cpp)
+#	
+#	target_compile_definitions(openshot-example-opencv PRIVATE
+#	                           -DTEST_MEDIA_PATH="${TEST_MEDIA_PATH}" )
+#	# Link test executable to the new library
+#	target_link_libraries(openshot-example-opencv openshot)
+#endif()

examples/Example_opencv.cpp

@@ -31,12 +31,15 @@
 #include <fstream>
 #include <iostream>
 #include <memory>
-#include "../../include/CVTracker.h"
-#include "../../include/CVStabilization.h"
-#include "../../include/CVObjectDetection.h"
+#include "CVTracker.h"
+#include "CVStabilization.h"
+#include "CVObjectDetection.h"
 
-#include "../../include/OpenShot.h"
-#include "../../include/CrashHandler.h"
+#include "Clip.h"
+#include "EffectBase.h"
+#include "EffectInfo.h"
+#include "Frame.h"
+#include "CrashHandler.h"
 
 using namespace openshot;
 using namespace std;
@@ -60,7 +63,7 @@ void displayClip(openshot::Clip &r9){
 
     // Opencv display window
     cv::namedWindow("Display Image", cv::WINDOW_NORMAL );
-    
+
     // Get video lenght
     int videoLenght = r9.Reader()->info.video_length;
 
@@ -118,7 +121,7 @@ int main(int argc, char* argv[]) {
         CVTracker tracker(trackerJson(r, false), processingController);
 
         // Start the tracking
-        tracker.trackClip(r9, 0, 100, true);
+        tracker.trackClip(r9, 0, 0, true);
         // Save the tracked data
         tracker.SaveTrackedData();
 
@@ -223,22 +226,23 @@ string jsonFormat(string key, string value, string type){
 // Return JSON string for the tracker effect
 string trackerJson(cv::Rect2d r, bool onlyProtoPath){
 
-    // Define path to save tracked data 
+    // Define path to save tracked data
     string protobufDataPath = "kcf_tracker.data";
     // Set the tracker
     string tracker = "KCF";
 
     // Construct all the composition of the JSON string
     string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath);
-    string trackerType = jsonFormat("tracker_type", tracker);
+    string trackerType = jsonFormat("tracker-type", tracker);
     string bboxCoords = jsonFormat(
-                                    "bbox", 
-                                            "{" + jsonFormat("x", to_string(r.x), "int") + 
-                                            "," + jsonFormat("y", to_string(r.y), "int") + 
-                                            "," + jsonFormat("w", to_string(r.width), "int") +
-                                            "," + jsonFormat("h", to_string(r.height), "int") + 
+                                    "region",
+                                            "{" + jsonFormat("x", to_string(r.x), "int") +
+                                            "," + jsonFormat("y", to_string(r.y), "int") +
+                                            "," + jsonFormat("width", to_string(r.width), "int") +
+                                            "," + jsonFormat("height", to_string(r.height), "int") +
+                                            "," + jsonFormat("first-frame", to_string(0), "int") +
                                             "}",
-                                    "rstring");  
+                                    "rstring");
 
     // Return only the the protobuf path in JSON format
     if(onlyProtoPath)
@@ -251,7 +255,7 @@ string trackerJson(cv::Rect2d r, bool onlyProtoPath){
 // Return JSON string for the stabilizer effect
 string stabilizerJson(bool onlyProtoPath){
 
-    // Define path to save stabilized data 
+    // Define path to save stabilized data
     string protobufDataPath = "example_stabilizer.data";
     // Set smoothing window value
     string smoothingWindow = "30";
@@ -270,13 +274,13 @@ string stabilizerJson(bool onlyProtoPath){
 
 string objectDetectionJson(bool onlyProtoPath){
 
-    // Define path to save object detection data 
+    // Define path to save object detection data
     string protobufDataPath = "example_object_detection.data";
     // Define processing device
     string processingDevice = "GPU";
     // Set path to model configuration file
     string modelConfiguration = "yolov3.cfg";
-    // Set path to model weights 
+    // Set path to model weights
     string modelWeights = "yolov3.weights";
     // Set path to class names file
     string classesFile = "obj.names";

src/CMakeLists.txt

@@ -86,6 +86,7 @@ set(OPENSHOT_SOURCES
   FrameMapper.cpp
   Json.cpp
   KeyFrame.cpp
+  TrackedObjectBase.cpp
   OpenShotVersion.cpp
   ZmqLogger.cpp
   PlayerBase.cpp
@@ -105,6 +106,7 @@ set(OPENSHOT_CV_SOURCES
   CVStabilization.cpp
   ClipProcessingJobs.cpp
   CVObjectDetection.cpp
+  TrackedObjectBBox.cpp
   effects/Stabilizer.cpp
   effects/Tracker.cpp
   effects/ObjectDetection.cpp
@@ -288,7 +290,7 @@ mark_as_advanced(QT_VERSION_STR)
 # Find FFmpeg libraries (used for video encoding / decoding)
 find_package(FFmpeg REQUIRED COMPONENTS avcodec avformat avutil swscale)
 
-set(all_comps avcodec avformat avutil swscale)
+set(all_comps avcodec avformat avutil swscale avresample)
 if(TARGET FFmpeg::swresample)
   list(APPEND all_comps swresample)
 else()
@@ -507,3 +509,4 @@ endif()
 set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Jonathan Thomas") #required
 
 include(CPack)
+

src/CVObjectDetection.cpp

@@ -2,6 +2,7 @@
  * @file
  * @brief Source file for CVObjectDetection class
  * @author Jonathan Thomas <jonathan@openshot.org>
+ * @author Brenno Caldato <brenno.caldato@outlook.com>
  *
  * @ref License
  */
@@ -82,10 +83,10 @@ void CVObjectDetection::detectObjectsClip(openshot::Clip &video, size_t _start,
     setProcessingDevice();
 
     size_t frame_number;
-    if(!process_interval || end == 0 || end-start == 0){
+    if(!process_interval || end <= 1 || end-start == 0){
         // Get total number of frames in video
-        start = video.Start() * video.Reader()->info.fps.ToInt();
-        end = video.End() * video.Reader()->info.fps.ToInt();
+        start = (int)(video.Start() * video.Reader()->info.fps.ToFloat());
+        end = (int)(video.End() * video.Reader()->info.fps.ToFloat());
     }
 
     for (frame_number = start; frame_number <= end; frame_number++)
@@ -105,7 +106,6 @@ void CVObjectDetection::detectObjectsClip(openshot::Clip &video, size_t _start,
         // Update progress
         processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
 
-        // std::cout<<"Frame: "<<frame_number<<"\n";
     }
 }
 
@@ -138,6 +138,7 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
     std::vector<int> classIds;
     std::vector<float> confidences;
     std::vector<cv::Rect> boxes;
+    std::vector<int> objectIds;
 
     for (size_t i = 0; i < outs.size(); ++i)
     {
@@ -180,13 +181,14 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
     sort.update(sortBoxes, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2)), confidences, classIds);
 
     // Clear data vectors
-    boxes.clear(); confidences.clear(); classIds.clear();
+    boxes.clear(); confidences.clear(); classIds.clear(); objectIds.clear();
     // Get SORT predicted boxes
     for(auto TBox : sort.frameTrackingResult){
         if(TBox.frame == frameId){
             boxes.push_back(TBox.box);
             confidences.push_back(TBox.confidence);
             classIds.push_back(TBox.classId);
+            objectIds.push_back(TBox.id);
         }
     }
 
@@ -202,12 +204,14 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
                         boxes.erase(boxes.begin() + j);
                         classIds.erase(classIds.begin() + j);
                         confidences.erase(confidences.begin() + j);
+                        objectIds.erase(objectIds.begin() + j);
                         break;
                     }
                     else{
                         boxes.erase(boxes.begin() + i);
                         classIds.erase(classIds.begin() + i);
                         confidences.erase(confidences.begin() + i);
+                        objectIds.erase(objectIds.begin() + i);
                         i = 0;
                         break;
                     }
@@ -226,12 +230,14 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
                         boxes.erase(boxes.begin() + j);
                         classIds.erase(classIds.begin() + j);
                         confidences.erase(confidences.begin() + j);
+                        objectIds.erase(objectIds.begin() + j);
                         break;
                     }
                     else{
                         boxes.erase(boxes.begin() + i);
                         classIds.erase(classIds.begin() + i);
                         confidences.erase(confidences.begin() + i);
+                        objectIds.erase(objectIds.begin() + i);
                         i = 0;
                         break;
                     }
@@ -251,7 +257,7 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
         normalized_boxes.push_back(normalized_box);
     }
 
-    detectionsData[frameId] = CVDetectionData(classIds, confidences, normalized_boxes, frameId);
+    detectionsData[frameId] = CVDetectionData(classIds, confidences, normalized_boxes, frameId, objectIds);
 }
 
 // Compute IOU between 2 boxes
@@ -359,6 +365,7 @@ void CVObjectDetection::AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, CV
         box->set_h(dData.boxes.at(i).height);
         box->set_classid(dData.classIds.at(i));
         box->set_confidence(dData.confidences.at(i));
+        box->set_objectid(dData.objectIds.at(i));
 
     }
 }
@@ -461,7 +468,10 @@ bool CVObjectDetection::_LoadObjDetectdData(){
         const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
 
         // Construct data vectors related to detections in the current frame
-        std::vector<int> classIds; std::vector<float> confidences; std::vector<cv::Rect_<float>> boxes;
+        std::vector<int> classIds; 
+        std::vector<float> confidences; 
+        std::vector<cv::Rect_<float>> boxes;
+        std::vector<int> objectIds;
 
         for(int i = 0; i < pbFrameData.bounding_box_size(); i++){
             // Get bounding box coordinates
@@ -472,13 +482,15 @@ bool CVObjectDetection::_LoadObjDetectdData(){
 
             // Get class Id (which will be assign to a class name) and prediction confidence
             int classId = pBox.Get(i).classid(); float confidence = pBox.Get(i).confidence();
+            // Get object Id
+            int objectId = pBox.Get(i).objectid();
 
             // Push back data into vectors
             boxes.push_back(box); classIds.push_back(classId); confidences.push_back(confidence);
         }
 
         // Assign data to object detector map
-        detectionsData[id] = CVDetectionData(classIds, confidences, boxes, id);
+        detectionsData[id] = CVDetectionData(classIds, confidences, boxes, id, objectIds);
     }
 
     // Delete all global objects allocated by libprotobuf.

src/CVObjectDetection.h

@@ -2,6 +2,7 @@
  * @file
  * @brief Header file for CVObjectDetection class
  * @author Jonathan Thomas <jonathan@openshot.org>
+ * @author Brenno Caldato <brenno.caldato@outlook.com>
  *
  * @ref License
  */
@@ -49,16 +50,24 @@ namespace openshot
     // Stores the detected object bounding boxes and its properties.
     struct CVDetectionData{
         CVDetectionData(){}
-        CVDetectionData(std::vector<int> _classIds, std::vector<float> _confidences, std::vector<cv::Rect_<float>> _boxes, size_t _frameId){
+        CVDetectionData(
+            std::vector<int> _classIds, 
+            std::vector<float> _confidences,
+            std::vector<cv::Rect_<float>> _boxes, 
+            size_t _frameId,
+            std::vector<int> _objectIds)
+        {
             classIds = _classIds;
             confidences = _confidences;
             boxes = _boxes;
             frameId = _frameId;
+            objectIds = _objectIds;
         }
         size_t frameId;
         std::vector<int> classIds;
         std::vector<float> confidences;
         std::vector<cv::Rect_<float>> boxes;
+        std::vector<int> objectIds;
     };
 
     /**

src/CVStabilization.cpp

@@ -2,6 +2,7 @@
  * @file
  * @brief Source file for CVStabilization class
  * @author Jonathan Thomas <jonathan@openshot.org>
+ * @author Brenno Caldato <brenno.caldato@outlook.com>
  *
  * @ref License
  */
@@ -43,8 +44,8 @@ using google::protobuf::util::TimeUtil;
 CVStabilization::CVStabilization(std::string processInfoJson, ProcessingController &processingController)
 : processingController(&processingController){
     SetJson(processInfoJson);
-    start = 0;
-    end = 0;
+    start = 1;
+    end = 1;
 }
 
 // Process clip and store necessary stabilization data
@@ -64,10 +65,10 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t
     cv::Size readerDims(video.Reader()->info.width, video.Reader()->info.height);
 
     size_t frame_number;
-    if(!process_interval || end == 0 || end-start == 0){
+    if(!process_interval || end <= 1 || end-start == 0){
         // Get total number of frames in video
-        start = video.Start() * video.Reader()->info.fps.ToInt();
-        end = video.End() * video.Reader()->info.fps.ToInt();
+        start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
+        end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
     }
 
     // Extract and track opticalflow features for each frame

src/CVStabilization.h

@@ -2,6 +2,7 @@
  * @file
  * @brief Header file for CVStabilization class
  * @author Jonathan Thomas <jonathan@openshot.org>
+ * @author Brenno Caldato <brenno.caldato@outlook.com>
  *
  * @ref License
  */