Skip to content

Commit

Permalink
ObjectDetection: updated object selection and transform handler
Browse files Browse the repository at this point in the history 
Removed the necessity to append the detected object index (related to the frame) to the effect JSON - which makes the JSON smaller and the performance better.
  • Loading branch information
@BrennoCaldato
BrennoCaldato committed Jan 27, 2021
1 parent 4a28654 commit a94f09d
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 80 deletions.
1 change: 1 addition & 0 deletions src/EffectBase.cpp
View file
Expand Up @@ -92,6 +92,7 @@ Json::Value EffectBase::JsonValue() const {
root["description"] = info.description;
root["has_video"] = info.has_video;
root["has_audio"] = info.has_audio;
root["has_tracked_object"] = info.has_tracked_object;
root["order"] = Order();

// return JsonValue
Expand Down
3 changes: 3 additions & 0 deletions src/EffectBase.h
View file
Expand Up @@ -97,6 +97,9 @@ namespace openshot
/// Set parent clip object of this effect
void ParentClip(openshot::ClipBase* new_clip);

/// Get the indexes and IDs of all visible objects in the given frame
virtual std::string GetVisibleObjects(int64_t frame_number) const {return {}; };

/// Get and Set JSON methods
virtual std::string Json() const = 0; ///< Generate JSON string of this object
virtual void SetJson(const std::string value) = 0; ///< Load JSON string into this object
Expand Down
102 changes: 57 additions & 45 deletions src/effects/ObjectDetection.cpp
View file
Expand Up @@ -28,8 +28,6 @@
* along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
*/

#include <string>

#include "effects/ObjectDetection.h"
#include "effects/Tracker.h"

Expand Down Expand Up @@ -264,6 +262,28 @@ DetectionData ObjectDetection::GetTrackedData(size_t frameId){

}

// Get the indexes and IDs of all visible objects in the given frame
std::string ObjectDetection::GetVisibleObjects(int64_t frame_number) const{

// Initialize the JSON objects
Json::Value root;
root["visible_objects_index"] = Json::Value(Json::arrayValue);
root["visible_objects_id"] = Json::Value(Json::arrayValue);

// Iterate through the tracked objects
for (const auto& trackedObject : trackedObjects){
// Get the tracked object JSON properties for this frame
Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
if (trackedObjectJSON["visible"]["value"].asBool()){
// Save the object's index and ID if it's visible in this frame
root["visible_objects_index"].append(trackedObject.first);
root["visible_objects_id"].append(trackedObject.second->Id());
}
}

return root.toStyledString();
}

// Generate JSON string of this object
std::string ObjectDetection::Json() const {

Expand All @@ -279,21 +299,22 @@ Json::Value ObjectDetection::JsonValue() const {
root["type"] = info.class_name;
root["protobuf_data_path"] = protobuf_data_path;
root["selected_object_index"] = selectedObjectIndex;

root["objects_id"] = Json::Value(Json::arrayValue);

for (auto const& trackedObject : trackedObjects){
Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"];
root["objects_id"].append(trackedObject.second->Id());
}

// Add the selected object Json to root
auto selectedObject = trackedObjects.at(selectedObjectIndex);
if (selectedObject){
Json::Value selectedObjectJSON = selectedObject->JsonValue();
root["delta_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_x"];
root["delta_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_y"];
root["scale_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_x"];
root["scale_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_y"];
root["rotation-"+to_string(selectedObjectIndex)] = selectedObjectJSON["rotation"];
root["delta_x"] = selectedObjectJSON["delta_x"];
root["delta_y"] = selectedObjectJSON["delta_y"];
root["scale_x"] = selectedObjectJSON["scale_x"];
root["scale_y"] = selectedObjectJSON["scale_y"];
root["rotation"] = selectedObjectJSON["rotation"];
}

// return JsonValue
Expand Down Expand Up @@ -336,26 +357,26 @@ void ObjectDetection::SetJsonValue(const Json::Value root) {
if (!root["selected_object_index"].isNull())
selectedObjectIndex = root["selected_object_index"].asInt();

for (auto const& trackedObject : trackedObjects){
// Set the object's ids
if (!root["objects_id"].isNull()){
for (auto const& trackedObject : trackedObjects){
Json::Value trackedObjectJSON;
trackedObjectJSON["box_id"] = root["box_id-"+to_string(trackedObject.first)];
trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
trackedObject.second->SetJsonValue(trackedObjectJSON);
}
}

// Set the selectec object's properties
if (!root["box_id-"+to_string(selectedObjectIndex)].isNull()){
Json::Value selectedObjectJSON;
selectedObjectJSON["box_id"] = root["box_id-"+to_string(selectedObjectIndex)];
selectedObjectJSON["delta_x"] = root["delta_x-"+to_string(selectedObjectIndex)];
selectedObjectJSON["delta_y"] = root["delta_y-"+to_string(selectedObjectIndex)];
selectedObjectJSON["scale_x"] = root["scale_x-"+to_string(selectedObjectIndex)];
selectedObjectJSON["scale_y"] = root["scale_y-"+to_string(selectedObjectIndex)];
selectedObjectJSON["rotation"] = root["rotation-"+to_string(selectedObjectIndex)];
if (!selectedObjectJSON.isNull()){
auto selectedObject = trackedObjects.at(selectedObjectIndex);
if (selectedObject)
selectedObject->SetJsonValue(selectedObjectJSON);
}
// Set the selected object's properties
Json::Value selectedObjectJSON;
selectedObjectJSON["delta_x"] = root["delta_x"];
selectedObjectJSON["delta_y"] = root["delta_y"];
selectedObjectJSON["scale_x"] = root["scale_x"];
selectedObjectJSON["scale_y"] = root["scale_y"];
selectedObjectJSON["rotation"] = root["rotation"];
if (!selectedObjectJSON.isNull()){
auto selectedObject = trackedObjects.at(selectedObjectIndex);
if (selectedObject)
selectedObject->SetJsonValue(selectedObjectJSON);
}
}

Expand All @@ -365,30 +386,21 @@ std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const {
// Generate JSON properties list
Json::Value root;

// root["visible_objects"] = Json::Value(Json::arrayValue);

for (auto const& trackedObject : trackedObjects){
Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
root["visible-"+to_string(trackedObject.first)] = trackedObjectJSON["visible"];
if (trackedObjectJSON["visible"]["value"].asBool())
root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"];
}

// Add the selected object Json to root
auto selectedObject = trackedObjects.at(selectedObjectIndex);
if (selectedObject){
Json::Value selectedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
root["box_id-"+to_string(selectedObjectIndex)] = selectedObjectJSON["box_id"];
root["visible-"+to_string(selectedObjectIndex)] = selectedObjectJSON["visible"];
root["x1-"+to_string(selectedObjectIndex)] = selectedObjectJSON["x1"];
root["y1-"+to_string(selectedObjectIndex)] = selectedObjectJSON["y1"];
root["x2-"+to_string(selectedObjectIndex)] = selectedObjectJSON["x2"];
root["y2-"+to_string(selectedObjectIndex)] = selectedObjectJSON["y2"];
root["delta_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_x"];
root["delta_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["delta_y"];
root["scale_x-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_x"];
root["scale_y-"+to_string(selectedObjectIndex)] = selectedObjectJSON["scale_y"];
root["rotation-"+to_string(selectedObjectIndex)] = selectedObjectJSON["rotation"];
root["box_id"] = selectedObjectJSON["box_id"];
root["visible"] = selectedObjectJSON["visible"];
root["x1"] = selectedObjectJSON["x1"];
root["y1"] = selectedObjectJSON["y1"];
root["x2"] = selectedObjectJSON["x2"];
root["y2"] = selectedObjectJSON["y2"];
root["delta_x"] = selectedObjectJSON["delta_x"];
root["delta_y"] = selectedObjectJSON["delta_y"];
root["scale_x"] = selectedObjectJSON["scale_x"];
root["scale_y"] = selectedObjectJSON["scale_y"];
root["rotation"] = selectedObjectJSON["rotation"];
}

root["selected_object_index"] = add_property_json("Selected Object", selectedObjectIndex, "int", "", NULL, 0, 200, false, requested_frame);
Expand Down
6 changes: 5 additions & 1 deletion src/effects/ObjectDetection.h
View file
Expand Up @@ -108,11 +108,15 @@ namespace openshot

std::shared_ptr<openshot::Frame> GetFrame(int64_t frame_number) override { return GetFrame(std::shared_ptr<Frame> (new Frame()), frame_number); }

// Load protobuf data file
/// Load protobuf data file
bool LoadObjDetectdData(std::string inputFilePath);

/// Get tracker info for the desired frame
DetectionData GetTrackedData(size_t frameId);

/// Get the indexes and IDs of all visible objects in the given frame
std::string GetVisibleObjects(int64_t frame_number) const override;

/// Get and Set JSON methods
std::string Json() const override; ///< Generate JSON string of this object
void SetJson(const std::string value) override; ///< Load JSON string into this object
Expand Down
97 changes: 63 additions & 34 deletions src/effects/Tracker.cpp
View file
Expand Up @@ -126,6 +126,28 @@ std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t f
return frame;
}

// Get the indexes and IDs of all visible objects in the given frame
std::string Tracker::GetVisibleObjects(int64_t frame_number) const{

// Initialize the JSON objects
Json::Value root;
root["visible_objects_index"] = Json::Value(Json::arrayValue);
root["visible_objects_id"] = Json::Value(Json::arrayValue);

// Iterate through the tracked objects
for (const auto& trackedObject : trackedObjects){
// Get the tracked object JSON properties for this frame
Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
if (trackedObjectJSON["visible"]["value"].asBool()){
// Save the object's index and ID if it's visible in this frame
root["visible_objects_index"].append(trackedObject.first);
root["visible_objects_id"].append(trackedObject.second->Id());
}
}

return root.toStyledString();
}

// Generate JSON string of this object
std::string Tracker::Json() const {

Expand All @@ -145,21 +167,19 @@ Json::Value Tracker::JsonValue() const {
root["BaseFPS"]["num"] = BaseFPS.num;
root["BaseFPS"]["den"] = BaseFPS.den;
root["TimeScale"] = this->TimeScale;
root["objects_id"] = Json::Value(Json::arrayValue);

// Get trackedData JSON
Json::Value trackedDataJSON;
trackedDataJSON = trackedData->JsonValue();

// Save the trackedData properties on root
// Add trackedObjects IDs to JSON
for (auto const& trackedObject : trackedObjects){
// Get the trackedObject JSON
Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
root["objects_id"].append(trackedObject.second->Id());
// Save the trackedObject JSON on root
root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"];
root["delta_x-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_x"];
root["delta_y-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_y"];
root["scale_x-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_x"];
root["scale_y-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_y"];
root["rotation-"+to_string(trackedObject.first)] = trackedObjectJSON["rotation"];
root["delta_x"] = trackedObjectJSON["delta_x"];
root["delta_y"] = trackedObjectJSON["delta_y"];
root["scale_x"] = trackedObjectJSON["scale_x"];
root["scale_y"] = trackedObjectJSON["scale_y"];
root["rotation"] = trackedObjectJSON["rotation"];
}

// return JsonValue
Expand Down Expand Up @@ -219,14 +239,22 @@ void Tracker::SetJsonValue(const Json::Value root) {
}
}

// Set the object's ids
if (!root["objects_id"].isNull()){
for (auto const& trackedObject : trackedObjects){
Json::Value trackedObjectJSON;
trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
trackedObject.second->SetJsonValue(trackedObjectJSON);
}
}

for (auto const& trackedObject : trackedObjects){
Json::Value trackedObjectJSON;
trackedObjectJSON["box_id"] = root["box_id-"+to_string(trackedObject.first)];
trackedObjectJSON["delta_x"] = root["delta_x-"+to_string(trackedObject.first)];
trackedObjectJSON["delta_y"] = root["delta_y-"+to_string(trackedObject.first)];
trackedObjectJSON["scale_x"] = root["scale_x-"+to_string(trackedObject.first)];
trackedObjectJSON["scale_y"] = root["scale_y-"+to_string(trackedObject.first)];
trackedObjectJSON["rotation"] = root["rotation-"+to_string(trackedObject.first)];
Json::Value trackedObjectJSON;
trackedObjectJSON["delta_x"] = root["delta_x"];
trackedObjectJSON["delta_y"] = root["delta_y"];
trackedObjectJSON["scale_x"] = root["scale_x"];
trackedObjectJSON["scale_y"] = root["scale_y"];
trackedObjectJSON["rotation"] = root["rotation"];
if (!trackedObjectJSON.isNull())
trackedObject.second->SetJsonValue(trackedObjectJSON);
}
Expand All @@ -239,22 +267,23 @@ void Tracker::SetJsonValue(const Json::Value root) {
std::string Tracker::PropertiesJSON(int64_t requested_frame) const {

// Generate JSON properties list
Json::Value root;
// Add trackedObjects properties to JSON
Json::Value root;

// Add trackedObjects IDs to JSON
for (auto const& trackedObject : trackedObjects){
Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
root["box_id-"+to_string(trackedObject.first)] = trackedObjectJSON["box_id"];
root["visible-"+to_string(trackedObject.first)] = trackedObjectJSON["visible"];
root["x1-"+to_string(trackedObject.first)] = trackedObjectJSON["x1"];
root["y1-"+to_string(trackedObject.first)] = trackedObjectJSON["y1"];
root["x2-"+to_string(trackedObject.first)] = trackedObjectJSON["x2"];
root["y2-"+to_string(trackedObject.first)] = trackedObjectJSON["y2"];
root["delta_x-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_x"];
root["delta_y-"+to_string(trackedObject.first)] = trackedObjectJSON["delta_y"];
root["scale_x-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_x"];
root["scale_y-"+to_string(trackedObject.first)] = trackedObjectJSON["scale_y"];
root["rotation-"+to_string(trackedObject.first)] = trackedObjectJSON["rotation"];
// Save the trackedObject Id on root
Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
root["box_id"] = trackedObjectJSON["box_id"];
root["visible"] = trackedObjectJSON["visible"];
root["x1"] = trackedObjectJSON["x1"];
root["y1"] = trackedObjectJSON["y1"];
root["x2"] = trackedObjectJSON["x2"];
root["y2"] = trackedObjectJSON["y2"];
root["delta_x"] = trackedObjectJSON["delta_x"];
root["delta_y"] = trackedObjectJSON["delta_y"];
root["scale_x"] = trackedObjectJSON["scale_x"];
root["scale_y"] = trackedObjectJSON["scale_y"];
root["rotation"] = trackedObjectJSON["rotation"];
}

// Append effect's properties
Expand All @@ -268,4 +297,4 @@ std::string Tracker::PropertiesJSON(int64_t requested_frame) const {

// Return formatted string
return root.toStyledString();
}
}
3 changes: 3 additions & 0 deletions src/effects/Tracker.h
View file
Expand Up @@ -85,6 +85,9 @@ namespace openshot
std::shared_ptr<Frame> GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number) override;
std::shared_ptr<openshot::Frame> GetFrame(int64_t frame_number) override { return GetFrame(std::shared_ptr<Frame> (new Frame()), frame_number); }

/// Get the indexes and IDs of all visible objects in the given frame
std::string GetVisibleObjects(int64_t frame_number) const override;

/// Get and Set JSON methods
std::string Json() const override; ///< Generate JSON string of this object
void SetJson(const std::string value) override; ///< Load JSON string into this object
Expand Down

0 comments on commit a94f09d

Please sign in to comment.