Added CVObjectDetection and ObjectDetection effect

Also included kalman filter functions and code for tracking the output boxes from DNN model

include/CVObjectDetection.h

@@ -0,0 +1,114 @@
+/**
+ * @file
+ * @brief Header file for CVObjectDetection class
+ * @author Jonathan Thomas <jonathan@openshot.org>
+ *
+ * @ref License
+ */
+
+/* LICENSE
+ *
+ * Copyright (c) 2008-2019 OpenShot Studios, LLC
+ * <http://www.openshotstudios.com/>. This file is part of
+ * OpenShot Library (libopenshot), an open-source project dedicated to
+ * delivering high quality video editing and animation solutions to the
+ * world. For more information visit <http://www.openshot.org/>.
+ *
+ * OpenShot Library (libopenshot) is free software: you can redistribute it
+ * and/or modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * OpenShot Library (libopenshot) is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <google/protobuf/util/time_util.h>
+
+#define int64 opencv_broken_int
+#define uint64 opencv_broken_uint
+#include <opencv2/dnn.hpp>
+#include <opencv2/opencv.hpp>
+#include <opencv2/core.hpp>
+#undef uint64
+#undef int64
+#include "Json.h"
+#include "ProcessingController.h"
+#include "Clip.h"
+#include "objdetectdata.pb.h"
+
+using google::protobuf::util::TimeUtil;
+
+struct CVDetectionData{
+    CVDetectionData(){}
+    CVDetectionData(std::vector<int> _classIds, std::vector<float> _confidences, std::vector<cv::Rect> _boxes, size_t _frameId){
+        classIds = _classIds;
+        confidences = _confidences;
+        boxes = _boxes;
+        frameId = _frameId;
+    }
+    size_t frameId;
+    std::vector<int> classIds;
+    std::vector<float> confidences;
+    std::vector<cv::Rect> boxes;
+};
+
+class CVObjectDetection{
+
+    private:
+
+    cv::dnn::Net net;
+    std::vector<std::string> classNames;
+    float confThreshold, nmsThreshold;
+
+    std::string classesFile;
+    std::string modelConfiguration;
+    std::string modelWeights;
+    std::string processingDevice;
+    std::string protobuf_data_path;
+
+    uint progress;
+
+    size_t start;
+    size_t end;
+
+    /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes
+	ProcessingController *processingController;
+
+    void setProcessingDevice();
+
+    void DetectObjects(const cv::Mat &frame, size_t frame_number);
+
+    // Remove the bounding boxes with low confidence using non-maxima suppression
+    void postprocess(const cv::Size &frameDims, const std::vector<cv::Mat>& out, size_t frame_number);
+
+    // Get the names of the output layers
+    std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net);
+
+    public:
+
+    std::map<size_t, CVDetectionData> detectionsData;
+
+    CVObjectDetection(std::string processInfoJson, ProcessingController &processingController);
+
+    void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false);
+
+    /// Protobuf Save and Load methods
+    // Save protobuf file
+    bool SaveTrackedData();
+    // Add frame object detection data into protobuf message.
+    void AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrameData, CVDetectionData& dData);
+    // Load protobuf file
+    bool LoadTrackedData();
+
+    /// Get and Set JSON methods
+    void SetJson(const std::string value); ///< Load JSON string into this object
+    void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
+};

include/CVTracker.h

@@ -19,6 +19,8 @@
 #include "ProcessingController.h"
 #include "trackerdata.pb.h"
 
+#include "../src/sort_filter/sort.hpp"
+
 using namespace std;
 using google::protobuf::util::TimeUtil;
 
@@ -49,6 +51,17 @@ struct FrameData{
   }
 };
 
+class RemoveJitter{
+  private:
+    std::vector<cv::Rect2d> bboxTracker;
+    int boxesInterval;
+    int boxesInVector;
+
+  public:
+    RemoveJitter(int boxesInterval);
+    void update(cv::Rect2d bbox, cv::Rect2d &out_bbox);
+};
+
 class CVTracker { 
   private:
     std::map<size_t, FrameData> trackedDataById; // Save tracked data       
@@ -67,12 +80,13 @@ class CVTracker {
     bool json_interval;
     size_t start;
     size_t end;
+
 
     // Initialize the tracker
     bool initTracker(cv::Mat &frame, size_t frameId);
 
     // Update the object tracker according to frame 
-    bool trackFrame(cv::Mat &frame, size_t frameId);
+    bool trackFrame(cv::Mat &frame, size_t frameId, SortTracker &sort, RemoveJitter &removeJitter);
 
   public:
 

include/ClipProcessingJobs.h

@@ -39,6 +39,7 @@
 
 	#include "CVStabilization.h"
     #include "CVTracker.h"
+	#include "CVObjectDetection.h"
 #endif
 
 #include <thread>
@@ -66,6 +67,8 @@ class ClipProcessingJobs{
 		void trackClip(Clip& clip, ProcessingController& controller);
 		// Apply stabilization to clip
 		void stabilizeClip(Clip& clip, ProcessingController& controller);
+		// Apply object detection to clip
+		void detectObjectsClip(Clip& clip, ProcessingController& controller);
 
 
 	public:

include/Effects.h

@@ -42,6 +42,7 @@
 #include "effects/Hue.h"
 #include "effects/Mask.h"
 #include "effects/Negate.h"
+#include "effects/ObjectDetection.h"
 #include "effects/Pixelate.h"
 #include "effects/Stabilizer.h"
 #include "effects/Saturation.h"

include/effects/ObjectDetection.h

@@ -0,0 +1,117 @@
+/**
+ * @file
+ * @brief Header file for Object Detection effect class
+ * @author Jonathan Thomas <jonathan@openshot.org>
+ *
+ * @ref License
+ */
+
+/* LICENSE
+ *
+ * Copyright (c) 2008-2019 OpenShot Studios, LLC
+ * <http://www.openshotstudios.com/>. This file is part of
+ * OpenShot Library (libopenshot), an open-source project dedicated to
+ * delivering high quality video editing and animation solutions to the
+ * world. For more information visit <http://www.openshot.org/>.
+ *
+ * OpenShot Library (libopenshot) is free software: you can redistribute it
+ * and/or modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * OpenShot Library (libopenshot) is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef OPENSHOT_OBJECT_DETECTION_EFFECT_H
+#define OPENSHOT_OBJECT_DETECTION_EFFECT_H
+
+#include "../EffectBase.h"
+
+#include <cmath>
+#include <stdio.h>
+#include <memory>
+#include "../Color.h"
+#include "../Json.h"
+#include "../KeyFrame.h"
+#include "../objdetectdata.pb.h"
+
+struct DetectionData{
+    DetectionData(){}
+    DetectionData(std::vector<int> _classIds, std::vector<float> _confidences, std::vector<cv::Rect> _boxes, size_t _frameId){
+        classIds = _classIds;
+        confidences = _confidences;
+        boxes = _boxes;
+        frameId = _frameId;
+    }
+    size_t frameId;
+    std::vector<int> classIds;
+    std::vector<float> confidences;
+    std::vector<cv::Rect> boxes;
+};
+
+namespace openshot
+{
+
+	/**
+	 * @brief This class stabilizes video clip to remove undesired shaking and jitter.
+	 *
+	 * Adding stabilization is useful to increase video quality overall, since it removes 
+	 * from subtle to harsh unexpected camera movements.
+	 */
+	class ObjectDetection : public EffectBase
+	{
+	private:
+		std::string protobuf_data_path;
+		std::map<size_t, DetectionData> detectionsData;
+		std::vector<std::string> classNames;
+
+		/// Init effect settings
+		void init_effect_details();
+
+		void drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame);
+
+	public:
+
+
+		/// Blank constructor, useful when using Json to load the effect properties
+		ObjectDetection(std::string clipTrackerDataPath);
+
+		/// Default constructor
+		ObjectDetection();
+
+		/// @brief This method is required for all derived classes of EffectBase, and returns a
+		/// modified openshot::Frame object
+		///
+		/// The frame object is passed into this method, and a frame_number is passed in which
+		/// tells the effect which settings to use from its keyframes (starting at 1).
+		///
+		/// @returns The modified openshot::Frame object
+		/// @param frame The frame object that needs the effect applied to it
+		/// @param frame_number The frame number (starting at 1) of the effect on the timeline.
+		std::shared_ptr<Frame> GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number) override;
+
+		// Load protobuf data file
+        bool LoadObjDetectdData(std::string inputFilePath);
+
+		DetectionData GetTrackedData(size_t frameId);
+
+		/// Get and Set JSON methods
+		std::string Json() const override; ///< Generate JSON string of this object
+		void SetJson(const std::string value) override; ///< Load JSON string into this object
+		Json::Value JsonValue() const override; ///< Generate Json::Value for this object
+		void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object
+
+		/// Get all properties for a specific frame (perfect for a UI to display the current state
+		/// of all properties at any time)
+		std::string PropertiesJSON(int64_t requested_frame) const override;
+	};
+
+}
+
+#endif

include/effects/Tracker.h

@@ -116,7 +116,7 @@ namespace openshot
         bool LoadTrackedData(std::string inputFilePath);
 
 		// Get tracker info for the desired frame 
-        EffectFrameData GetTrackedData(int frameId);
+        EffectFrameData GetTrackedData(size_t frameId);
 
 		/// Get and Set JSON methods
 		std::string Json() const override; ///< Generate JSON string of this object