From a94f09d105cebcea54b6132ef1ee0110dc73ff6c Mon Sep 17 00:00:00 2001 From: Brenno Date: Wed, 27 Jan 2021 17:18:23 -0300 Subject: [PATCH] ObjectDetection: updated object selection and transform handler Removed the necessity to append the detected object index (related to the frame) to the effect JSON - which makes the JSON smaller and the performance better. --- src/EffectBase.cpp | 1 + src/EffectBase.h | 3 + src/effects/ObjectDetection.cpp | 102 ++++++++++++++++++-------------- src/effects/ObjectDetection.h | 6 +- src/effects/Tracker.cpp | 97 +++++++++++++++++++----------- src/effects/Tracker.h | 3 + 6 files changed, 132 insertions(+), 80 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index aa2bc4e42..9a48c1c9b 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -92,6 +92,7 @@ Json::Value EffectBase::JsonValue() const { root["description"] = info.description; root["has_video"] = info.has_video; root["has_audio"] = info.has_audio; + root["has_tracked_object"] = info.has_tracked_object; root["order"] = Order(); // return JsonValue diff --git a/src/EffectBase.h b/src/EffectBase.h index 320e840cf..ae5cf257f 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -97,6 +97,9 @@ namespace openshot /// Set parent clip object of this effect void ParentClip(openshot::ClipBase* new_clip); + /// Get the indexes and IDs of all visible objects in the given frame + virtual std::string GetVisibleObjects(int64_t frame_number) const {return {}; }; + /// Get and Set JSON methods virtual std::string Json() const = 0; ///< Generate JSON string of this object virtual void SetJson(const std::string value) = 0; ///< Load JSON string into this object diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index a9ee04114..e11ae91ca 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -28,8 +28,6 @@ * along with OpenShot Library. If not, see . */ -#include - #include "effects/ObjectDetection.h" #include "effects/Tracker.h" @@ -264,6 +262,28 @@ DetectionData ObjectDetection::GetTrackedData(size_t frameId){ } +// Get the indexes and IDs of all visible objects in the given frame +std::string ObjectDetection::GetVisibleObjects(int64_t frame_number) const{ + + // Initialize the JSON objects + Json::Value root; + root["visible_objects_index"] = Json::Value(Json::arrayValue); + root["visible_objects_id"] = Json::Value(Json::arrayValue); + + // Iterate through the tracked objects + for (const auto& trackedObject : trackedObjects){ + // Get the tracked object JSON properties for this frame + Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number); + if (trackedObjectJSON["visible"]["value"].asBool()){ + // Save the object's index and ID if it's visible in this frame + root["visible_objects_index"].append(trackedObject.first); + root["visible_objects_id"].append(trackedObject.second->Id()); + } + } + + return root.toStyledString(); +} + // Generate JSON string of this object std::string ObjectDetection::Json() const { @@ -279,21 +299,22 @@ Json::Value ObjectDetection::JsonValue() const { root["type"] = info.class_name; root["protobuf_data_path"] = protobuf_data_path; root["selected_object_index"] = selectedObjectIndex; - + root["objects_id"] = Json::Value(Json::arrayValue); + for (auto const& trackedObject : trackedObjects){ Json::Value trackedObjectJSON = trackedObject.second->JsonValue(); - root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"]; + root["objects_id"].append(trackedObject.second->Id()); } // Add the selected object Json to root auto selectedObject = trackedObjects.at(selectedObjectIndex); if (selectedObject){ Json::Value selectedObjectJSON = selectedObject->JsonValue(); - root["delta_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_x"]; - root["delta_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_y"]; - root["scale_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_x"]; - root["scale_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_y"]; - root["rotation-"+to_string(selectedObjectIndex)] = selectedObjectJSON["rotation"]; + root["delta_x"] = selectedObjectJSON["delta_x"]; + root["delta_y"] = selectedObjectJSON["delta_y"]; + root["scale_x"] = selectedObjectJSON["scale_x"]; + root["scale_y"] = selectedObjectJSON["scale_y"]; + root["rotation"] = selectedObjectJSON["rotation"]; } // return JsonValue @@ -336,26 +357,26 @@ void ObjectDetection::SetJsonValue(const Json::Value root) { if (!root["selected_object_index"].isNull()) selectedObjectIndex = root["selected_object_index"].asInt(); - for (auto const& trackedObject : trackedObjects){ + // Set the object's ids + if (!root["objects_id"].isNull()){ + for (auto const& trackedObject : trackedObjects){ Json::Value trackedObjectJSON; - trackedObjectJSON["box_id"] = root["box_id-"+to_string(trackedObject.first)]; + trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString(); trackedObject.second->SetJsonValue(trackedObjectJSON); + } } - // Set the selectec object's properties - if (!root["box_id-"+to_string(selectedObjectIndex)].isNull()){ - Json::Value selectedObjectJSON; - selectedObjectJSON["box_id"] = root["box_id-"+to_string(selectedObjectIndex)]; - selectedObjectJSON["delta_x"] = root["delta_x-"+to_string(selectedObjectIndex)]; - selectedObjectJSON["delta_y"] = root["delta_y-"+to_string(selectedObjectIndex)]; - selectedObjectJSON["scale_x"] = root["scale_x-"+to_string(selectedObjectIndex)]; - selectedObjectJSON["scale_y"] = root["scale_y-"+to_string(selectedObjectIndex)]; - selectedObjectJSON["rotation"] = root["rotation-"+to_string(selectedObjectIndex)]; - if (!selectedObjectJSON.isNull()){ - auto selectedObject = trackedObjects.at(selectedObjectIndex); - if (selectedObject) - selectedObject->SetJsonValue(selectedObjectJSON); - } + // Set the selected object's properties + Json::Value selectedObjectJSON; + selectedObjectJSON["delta_x"] = root["delta_x"]; + selectedObjectJSON["delta_y"] = root["delta_y"]; + selectedObjectJSON["scale_x"] = root["scale_x"]; + selectedObjectJSON["scale_y"] = root["scale_y"]; + selectedObjectJSON["rotation"] = root["rotation"]; + if (!selectedObjectJSON.isNull()){ + auto selectedObject = trackedObjects.at(selectedObjectIndex); + if (selectedObject) + selectedObject->SetJsonValue(selectedObjectJSON); } } @@ -365,30 +386,21 @@ std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const { // Generate JSON properties list Json::Value root; - // root["visible_objects"] = Json::Value(Json::arrayValue); - - for (auto const& trackedObject : trackedObjects){ - Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame); - root["visible-"+to_string(trackedObject.first)] = trackedObjectJSON["visible"]; - if (trackedObjectJSON["visible"]["value"].asBool()) - root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"]; - } - // Add the selected object Json to root auto selectedObject = trackedObjects.at(selectedObjectIndex); if (selectedObject){ Json::Value selectedObjectJSON = selectedObject->PropertiesJSON(requested_frame); - root["box_id-"+to_string(selectedObjectIndex)] = selectedObjectJSON["box_id"]; - root["visible-"+to_string(selectedObjectIndex)] = selectedObjectJSON["visible"]; - root["x1-"+to_string(selectedObjectIndex)] = selectedObjectJSON["x1"]; - root["y1-"+to_string(selectedObjectIndex)] = selectedObjectJSON["y1"]; - root["x2-"+to_string(selectedObjectIndex)] = selectedObjectJSON["x2"]; - root["y2-"+to_string(selectedObjectIndex)] = selectedObjectJSON["y2"]; - root["delta_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_x"]; - root["delta_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_y"]; - root["scale_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_x"]; - root["scale_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_y"]; - root["rotation-"+to_string(selectedObjectIndex)] = selectedObjectJSON["rotation"]; + root["box_id"] = selectedObjectJSON["box_id"]; + root["visible"] = selectedObjectJSON["visible"]; + root["x1"] = selectedObjectJSON["x1"]; + root["y1"] = selectedObjectJSON["y1"]; + root["x2"] = selectedObjectJSON["x2"]; + root["y2"] = selectedObjectJSON["y2"]; + root["delta_x"] = selectedObjectJSON["delta_x"]; + root["delta_y"] = selectedObjectJSON["delta_y"]; + root["scale_x"] = selectedObjectJSON["scale_x"]; + root["scale_y"] = selectedObjectJSON["scale_y"]; + root["rotation"] = selectedObjectJSON["rotation"]; } root["selected_object_index"] = add_property_json("Selected Object", selectedObjectIndex, "int", "", NULL, 0, 200, false, requested_frame); diff --git a/src/effects/ObjectDetection.h b/src/effects/ObjectDetection.h index 39f7f33c0..4f1ad4810 100644 --- a/src/effects/ObjectDetection.h +++ b/src/effects/ObjectDetection.h @@ -108,11 +108,15 @@ namespace openshot std::shared_ptr GetFrame(int64_t frame_number) override { return GetFrame(std::shared_ptr (new Frame()), frame_number); } - // Load protobuf data file + /// Load protobuf data file bool LoadObjDetectdData(std::string inputFilePath); + /// Get tracker info for the desired frame DetectionData GetTrackedData(size_t frameId); + /// Get the indexes and IDs of all visible objects in the given frame + std::string GetVisibleObjects(int64_t frame_number) const override; + /// Get and Set JSON methods std::string Json() const override; ///< Generate JSON string of this object void SetJson(const std::string value) override; ///< Load JSON string into this object diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index ec7553929..f3b353571 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -126,6 +126,28 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f return frame; } +// Get the indexes and IDs of all visible objects in the given frame +std::string Tracker::GetVisibleObjects(int64_t frame_number) const{ + + // Initialize the JSON objects + Json::Value root; + root["visible_objects_index"] = Json::Value(Json::arrayValue); + root["visible_objects_id"] = Json::Value(Json::arrayValue); + + // Iterate through the tracked objects + for (const auto& trackedObject : trackedObjects){ + // Get the tracked object JSON properties for this frame + Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number); + if (trackedObjectJSON["visible"]["value"].asBool()){ + // Save the object's index and ID if it's visible in this frame + root["visible_objects_index"].append(trackedObject.first); + root["visible_objects_id"].append(trackedObject.second->Id()); + } + } + + return root.toStyledString(); +} + // Generate JSON string of this object std::string Tracker::Json() const { @@ -145,21 +167,19 @@ Json::Value Tracker::JsonValue() const { root["BaseFPS"]["num"] = BaseFPS.num; root["BaseFPS"]["den"] = BaseFPS.den; root["TimeScale"] = this->TimeScale; + root["objects_id"] = Json::Value(Json::arrayValue); - // Get trackedData JSON - Json::Value trackedDataJSON; - trackedDataJSON = trackedData->JsonValue(); - - // Save the trackedData properties on root + // Add trackedObjects IDs to JSON for (auto const& trackedObject : trackedObjects){ + // Get the trackedObject JSON Json::Value trackedObjectJSON = trackedObject.second->JsonValue(); + root["objects_id"].append(trackedObject.second->Id()); // Save the trackedObject JSON on root - root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"]; - root["delta_x-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_x"]; - root["delta_y-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_y"]; - root["scale_x-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_x"]; - root["scale_y-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_y"]; - root["rotation-"+to_string(trackedObject.first)] = trackedObjectJSON["rotation"]; + root["delta_x"] = trackedObjectJSON["delta_x"]; + root["delta_y"] = trackedObjectJSON["delta_y"]; + root["scale_x"] = trackedObjectJSON["scale_x"]; + root["scale_y"] = trackedObjectJSON["scale_y"]; + root["rotation"] = trackedObjectJSON["rotation"]; } // return JsonValue @@ -219,14 +239,22 @@ void Tracker::SetJsonValue(const Json::Value root) { } } + // Set the object's ids + if (!root["objects_id"].isNull()){ + for (auto const& trackedObject : trackedObjects){ + Json::Value trackedObjectJSON; + trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString(); + trackedObject.second->SetJsonValue(trackedObjectJSON); + } + } + for (auto const& trackedObject : trackedObjects){ - Json::Value trackedObjectJSON; - trackedObjectJSON["box_id"] = root["box_id-"+to_string(trackedObject.first)]; - trackedObjectJSON["delta_x"] = root["delta_x-"+to_string(trackedObject.first)]; - trackedObjectJSON["delta_y"] = root["delta_y-"+to_string(trackedObject.first)]; - trackedObjectJSON["scale_x"] = root["scale_x-"+to_string(trackedObject.first)]; - trackedObjectJSON["scale_y"] = root["scale_y-"+to_string(trackedObject.first)]; - trackedObjectJSON["rotation"] = root["rotation-"+to_string(trackedObject.first)]; + Json::Value trackedObjectJSON; + trackedObjectJSON["delta_x"] = root["delta_x"]; + trackedObjectJSON["delta_y"] = root["delta_y"]; + trackedObjectJSON["scale_x"] = root["scale_x"]; + trackedObjectJSON["scale_y"] = root["scale_y"]; + trackedObjectJSON["rotation"] = root["rotation"]; if (!trackedObjectJSON.isNull()) trackedObject.second->SetJsonValue(trackedObjectJSON); } @@ -239,22 +267,23 @@ void Tracker::SetJsonValue(const Json::Value root) { std::string Tracker::PropertiesJSON(int64_t requested_frame) const { // Generate JSON properties list - Json::Value root; - - // Add trackedObjects properties to JSON + Json::Value root; + + // Add trackedObjects IDs to JSON for (auto const& trackedObject : trackedObjects){ - Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame); - root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"]; - root["visible-"+to_string(trackedObject.first)] = trackedObjectJSON["visible"]; - root["x1-"+to_string(trackedObject.first)] = trackedObjectJSON["x1"]; - root["y1-"+to_string(trackedObject.first)] = trackedObjectJSON["y1"]; - root["x2-"+to_string(trackedObject.first)] = trackedObjectJSON["x2"]; - root["y2-"+to_string(trackedObject.first)] = trackedObjectJSON["y2"]; - root["delta_x-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_x"]; - root["delta_y-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_y"]; - root["scale_x-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_x"]; - root["scale_y-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_y"]; - root["rotation-"+to_string(trackedObject.first)] = trackedObjectJSON["rotation"]; + // Save the trackedObject Id on root + Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame); + root["box_id"] = trackedObjectJSON["box_id"]; + root["visible"] = trackedObjectJSON["visible"]; + root["x1"] = trackedObjectJSON["x1"]; + root["y1"] = trackedObjectJSON["y1"]; + root["x2"] = trackedObjectJSON["x2"]; + root["y2"] = trackedObjectJSON["y2"]; + root["delta_x"] = trackedObjectJSON["delta_x"]; + root["delta_y"] = trackedObjectJSON["delta_y"]; + root["scale_x"] = trackedObjectJSON["scale_x"]; + root["scale_y"] = trackedObjectJSON["scale_y"]; + root["rotation"] = trackedObjectJSON["rotation"]; } // Append effect's properties @@ -268,4 +297,4 @@ std::string Tracker::PropertiesJSON(int64_t requested_frame) const { // Return formatted string return root.toStyledString(); -} +} \ No newline at end of file diff --git a/src/effects/Tracker.h b/src/effects/Tracker.h index e5cca0477..e52100941 100644 --- a/src/effects/Tracker.h +++ b/src/effects/Tracker.h @@ -85,6 +85,9 @@ namespace openshot std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; std::shared_ptr GetFrame(int64_t frame_number) override { return GetFrame(std::shared_ptr (new Frame()), frame_number); } + /// Get the indexes and IDs of all visible objects in the given frame + std::string GetVisibleObjects(int64_t frame_number) const override; + /// Get and Set JSON methods std::string Json() const override; ///< Generate JSON string of this object void SetJson(const std::string value) override; ///< Load JSON string into this object