From 1215d042c7c741a49c85090bb1d827f11cc26be7 Mon Sep 17 00:00:00 2001 From: Brenno Date: Thu, 12 Nov 2020 21:25:27 -0300 Subject: [PATCH] Added scale factor to class to replace OpFPS. --- src/KeyFrameBBox.cpp | 253 ++++++++++++++++++++++++++++----------- src/KeyFrameBBox.h | 125 ++++++++++++++----- src/effects/Tracker.cpp | 56 +++++---- src/effects/Tracker.h | 8 +- tests/KeyFrame_Tests.cpp | 63 +++++++++- 5 files changed, 383 insertions(+), 122 deletions(-) diff --git a/src/KeyFrameBBox.cpp b/src/KeyFrameBBox.cpp index c0d3edcc5..7bb0bebb4 100644 --- a/src/KeyFrameBBox.cpp +++ b/src/KeyFrameBBox.cpp @@ -115,8 +115,8 @@ namespace { } -KeyFrameBBox::KeyFrameBBox(): delta_x(0.0), delta_y(0.0), scale_x(0.0), scale_y(0.0) { - +KeyFrameBBox::KeyFrameBBox(): delta_x(0.0), delta_y(0.0), scale_x(0.0), scale_y(0.0), rotation(0.0) { + this->TimeScale = 1.0; return; } @@ -124,7 +124,7 @@ void KeyFrameBBox::AddDisplacement(int64_t frame_num, double _delta_x, double _d if (!this->Contains((int64_t) frame_num)) return; - double time = this->FrameNToTime(frame_num); + double time = this->FrameNToTime(frame_num, 1.0); if (_delta_x != 0.0) delta_x.AddPoint(time, _delta_x, openshot::InterpolationType::LINEAR); @@ -138,7 +138,7 @@ void KeyFrameBBox::AddScale(int64_t frame_num, double _scale_x, double _scale_y) if (!this->Contains((double) frame_num)) return; - double time = this->FrameNToTime(frame_num); + double time = this->FrameNToTime(frame_num, 1.0); if (_scale_x != 0.0) scale_x.AddPoint(time, _scale_x, openshot::InterpolationType::LINEAR); @@ -148,6 +148,15 @@ void KeyFrameBBox::AddScale(int64_t frame_num, double _scale_x, double _scale_y) return; } +void KeyFrameBBox::AddRotation(int64_t _frame_num, double rot){ + if (!this->Contains((double) _frame_num)) + return; + + double time = this->FrameNToTime(_frame_num, 1.0); + + rotation.AddPoint(time, rot, openshot::InterpolationType::LINEAR); +} + void KeyFrameBBox::AddBox(int64_t _frame_num , float _cx, float _cy, float _width, float _height){ if (_frame_num < 0) @@ -155,7 +164,7 @@ void KeyFrameBBox::AddBox(int64_t _frame_num , float _cx, float _cy, float _widt BBox box = BBox(_cx, _cy, _width, _height); - double time = this->FrameNToTime(_frame_num); + double time = this->FrameNToTime(_frame_num, 1.0); auto it = BoxVec.find(time); if (it != BoxVec.end()) @@ -176,45 +185,61 @@ int64_t KeyFrameBBox::GetLength() const{ bool KeyFrameBBox::Contains(int64_t frame_num) { - double time = this->FrameNToTime(frame_num); + double time = this->FrameNToTime(frame_num, 1.0); - auto it = BoxVec.find(time); - if (it != BoxVec.end()) - return true; - - return false; + auto it = BoxVec.lower_bound(time); + if (it == BoxVec.end()) + return false; + + return true; } void KeyFrameBBox::RemovePoint(int64_t frame_number){ - double time = this->FrameNToTime(frame_number); + double time = this->FrameNToTime(frame_number, 1.0); auto it = BoxVec.find(time); if (it != BoxVec.end()){ - BoxVec.erase(frame_number); + BoxVec.erase(time); - RemoveDelta(frame_number); - RemoveScale(frame_number); + RemoveDelta(time); + RemoveScale(time); } return; } void KeyFrameBBox::RemoveDelta(int64_t frame_number) { - double attr_x = this->delta_x.GetValue(frame_number); - Point point_x = this->delta_x.GetClosestPoint(Point((double) frame_number, attr_x)); - if (point_x.co.X == (double) frame_number) + double time = this->FrameNToTime(frame_number, 1.0); + + double attr_x = this->delta_x.GetValue(time); + Point point_x = this->delta_x.GetClosestPoint(Point((double) time, attr_x)); + if (point_x.co.X == (double) time) this->delta_x.RemovePoint(point_x); - double attr_y = this->delta_y.GetValue(frame_number); - Point point_y = this->delta_y.GetClosestPoint(Point((double) frame_number, attr_y)); - if (point_y.co.X == (double) frame_number) + double attr_y = this->delta_y.GetValue(time); + Point point_y = this->delta_y.GetClosestPoint(Point((double) time, attr_y)); + if (point_y.co.X == (double) time) this->delta_y.RemovePoint(point_y); return; } +void KeyFrameBBox::RemoveRotation(int64_t frame_number) { + + double time = this->FrameNToTime(frame_number, 1.0); + + double rot = this->rotation.GetValue(time); + Point point_rot = this->rotation.GetClosestPoint(Point((double) time, rot)); + if (point_rot.co.X == (double) time) + this->rotation.RemovePoint(point_rot); + + return; +} + + + void KeyFrameBBox::PrintParams() { std::cout << "delta_x "; this->delta_x.PrintPoints(); @@ -227,51 +252,40 @@ void KeyFrameBBox::PrintParams() { std::cout << "scale_y "; this->scale_y.PrintPoints(); + + std::cout << "rotation "; + this->rotation.PrintPoints(); + } void KeyFrameBBox::RemoveScale(int64_t frame_number) { - double attr_x = this->scale_x.GetValue(frame_number); - Point point_x = this->scale_x.GetClosestPoint(Point((double) frame_number, attr_x)); - if (point_x.co.X == (double) frame_number) + double time = this->FrameNToTime(frame_number, 1.0); + + double attr_x = this->scale_x.GetValue(time); + Point point_x = this->scale_x.GetClosestPoint(Point((double) time, attr_x)); + if (point_x.co.X == (double) time) this->scale_x.RemovePoint(point_x); - double attr_y = this->scale_y.GetValue(frame_number); - Point point_y = this->scale_y.GetClosestPoint(Point((double) frame_number, attr_y)); - if (point_y.co.X == (double) frame_number) + double attr_y = this->scale_y.GetValue(time); + Point point_y = this->scale_y.GetClosestPoint(Point((double) time, attr_y)); + if (point_y.co.X == (double) time) this->scale_y.RemovePoint(point_y); return; } -/*BBox KeyFrameBBox::GetValue(int64_t frame_number){ - - double time = this->FrameNToTime(frame_number); - - auto it = BoxVec.find(time); - if (it != BoxVec.end()){ - BBox res = it->second; - res.cx += this->delta_x.GetValue(time); - res.cy += this->delta_y.GetValue(time); - res.height += this->scale_y.GetValue(time); - res.width += this->scale_x.GetValue(time); - - return res; - } else { - - - } - - BBox val; - - return val; -}*/ BBox KeyFrameBBox::GetValue(int64_t frame_number){ - double time = this->FrameNToTime(frame_number); + double time = this->FrameNToTime(frame_number, this->TimeScale); auto it = BoxVec.lower_bound(time); + if (it == BoxVec.end()){ + BBox resp; + return resp; + } + if (it->first == time){ BBox res = it->second; res.cx += this->delta_x.GetValue(time); @@ -280,20 +294,22 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number){ res.width += this->scale_x.GetValue(time); return res; - } else { - BBox second_ref = it->second; - //advance(it, -1); - BBox first_ref = prev(it, 1)->second; + } + + BBox second_ref = it->second; + BBox first_ref = prev(it, 1)->second; - BBox res = InterpolateBoxes(prev(it, 1)->first, it->first, first_ref, second_ref, time); + BBox res = InterpolateBoxes(prev(it, 1)->first, it->first, first_ref, second_ref, time); - res.cx += this->delta_x.GetValue(time); - res.cy += this->delta_y.GetValue(time); - res.height += this->scale_y.GetValue(time); - res.width += this->scale_x.GetValue(time); + /*later add rotation transform to these points*/ + + res.cx += this->delta_x.GetValue(time); + res.cy += this->delta_y.GetValue(time); + res.height += this->scale_y.GetValue(time); + res.width += this->scale_x.GetValue(time); - return res; - } + return res; + } @@ -325,17 +341,120 @@ BBox KeyFrameBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, } -void KeyFrameBBox::SetFPS(Fraction fps){ - this->fps = fps; +void KeyFrameBBox::SetBaseFPS(Fraction fps){ + this->BaseFps = fps; return; } -Fraction KeyFrameBBox::GetFPS(){ - return fps; +Fraction KeyFrameBBox::GetBaseFPS(){ + return BaseFps; } -double KeyFrameBBox::FrameNToTime(int64_t frame_number){ - double time = ((double) frame_number) * this->fps.Reciprocal().ToDouble(); +double KeyFrameBBox::FrameNToTime(int64_t frame_number, double time_scale){ + double time = ((double) frame_number) * this->BaseFps.Reciprocal().ToDouble() * time_scale; return time; } + +void KeyFrameBBox::ScalePoints(double time_scale){ + this->TimeScale = time_scale; +} + + +// Generate JSON string of this object +std::string KeyFrameBBox::Json() { + + // Return formatted string + return JsonValue().toStyledString(); +} + +// Generate Json::Value for this object +Json::Value KeyFrameBBox::JsonValue() { + + // Create root json object + Json::Value root; + //root["Points"] = Json::Value(Json::arrayValue); + + root["BaseFPS"]["num"] = BaseFps.num; + root["BaseFPS"]["den"] = BaseFps.den; + root["TimeScale"] = this->TimeScale; + root["boxes"] = Json::Value(Json::arrayValue); + + // loop through points + for (auto const& x : BoxVec){ + Json::Value elem; + elem["key"] = x.first; + elem["val"] = x.second.JsonValue(); + root["boxes"].append(elem); + } + + + root["delta_x"] = delta_x.JsonValue(); + root["delta_y"] = delta_y.JsonValue(); + root["scale_x"] = scale_x.JsonValue(); + root["scale_y"] = scale_y.JsonValue(); + root["rotation"] = rotation.JsonValue(); + + // return JsonValue + return root; +} + +// Load JSON string into this object +void KeyFrameBBox::SetJson(const std::string value) { + + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } + + return; +} + +void KeyFrameBBox::clear(){ + BoxVec.clear(); +} + + + +// Load Json::Value into this object +void KeyFrameBBox::SetJsonValue(const Json::Value root) { + // Clear existing points + BoxVec.clear(); + + delta_x.SetJsonValue(root["delta_x"]); + delta_y.SetJsonValue(root["delta_y"]); + scale_x.SetJsonValue(root["scale_x"]); + scale_y.SetJsonValue(root["scale_y"]); + rotation.SetJsonValue(root["rotation"]); + + + if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject()) { + if (!root["BaseFPS"]["num"].isNull()) + BaseFps.num = (int) root["BaseFPS"]["num"].asInt(); + if (!root["BaseFPS"]["den"].isNull()) + BaseFps.den = (int) root["BaseFPS"]["den"].asInt(); + } + if (!root["TimeScale"].isNull()) { + this->TimeScale = (double) root["TimeScale"].asDouble(); + } + + if (!root["boxes"].isNull()){ + // loop through points + for (const auto existing_point : root["boxes"]) { + // Create Point + BBox box; + box.SetJsonValue(existing_point["val"]); + + BoxVec.insert({existing_point["key"].asDouble(), box}); + } + } + return; +} \ No newline at end of file diff --git a/src/KeyFrameBBox.h b/src/KeyFrameBBox.h index 9d3083183..a80c7d7d7 100644 --- a/src/KeyFrameBBox.h +++ b/src/KeyFrameBBox.h @@ -46,26 +46,6 @@ #include "KeyFrame.h" -struct BBox{ - float cx = -1; - float cy = -1; - float width = -1; - float height = -1; - - // Constructors - BBox(){ - return; - } - - BBox(float _cx, float _cy, float _width, float _height){ - //frame_num = _frame_num; - cx = _cx; - cy = _cy; - width = _width; - height = _height; - } -}; - namespace openshot { /** @@ -79,49 +59,130 @@ namespace openshot { * \endcode */ + + + struct BBox{ + float cx = -1; + float cy = -1; + float width = -1; + float height = -1; + + // Constructors + BBox(){ + return; + } + + BBox(float _cx, float _cy, float _width, float _height){ + //frame_num = _frame_num; + cx = _cx; + cy = _cy; + width = _width; + height = _height; + } + + std::string Json() const { + // Return formatted string + return JsonValue().toStyledString(); + } + + // Generate Json::Value for this object + Json::Value JsonValue() const { + + // Create root json object + Json::Value root; + root["cx"] = cx; + root["cy"] = cy; + root["height"] = height; + root["width"] = width; + + return root; + } + + // Load JSON string into this object + void SetJson(const std::string value) { + + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } + } + + // Load Json::Value into this object + void SetJsonValue(const Json::Value root) { + + // Set data from Json (if key is found) + if (!root["cx"].isNull()) + cx = root["cx"].asDouble(); + if (!root["cy"].isNull()) + cy = root["cy"].asDouble(); + if (!root["height"].isNull()) + height = root["height"].asDouble(); + if (!root["width"].isNull()) + width = root["width"].asDouble(); + } + }; + + class KeyFrameBBox { private: bool visible; - Fraction fps; + Fraction BaseFps; + double TimeScale; std::map BoxVec; public: Keyframe delta_x; Keyframe delta_y; Keyframe scale_x; Keyframe scale_y; - + Keyframe rotation; + KeyFrameBBox(); void AddDisplacement(int64_t _frame_num, double _delta_x, double _delta_y); void AddScale(int64_t _frame_num, double _delta_x, double _delta_y); void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height); - - void SetFPS(Fraction fps); - Fraction GetFPS(); + void AddRotation(int64_t _frame_num, double rot); + + void SetBaseFPS(Fraction fps); + Fraction GetBaseFPS(); + + void ScalePoints(double scale); bool Contains(int64_t frame_number); //double GetDelta(int64_t index) const ; int64_t GetLength() const; - - /// Get and Set JSON methods - //std::string Json() const ; ///< Generate JSON string of this object - //Json::Value JsonValue() const ; ///< Generate Json::Value for this object - //void SetJson(const std::string value) ; ///< Load JSON string into this object - //void SetJsonValue(const Json::Value root) ; ///< Load Json::Value into this object /// Remove a points by frame_number void RemovePoint(int64_t frame_number); void RemoveDelta(int64_t frame_number); void RemoveScale(int64_t frame_number); + void RemoveRotation(int64_t frame_number); BBox GetValue(int64_t frame_number); /// Print collection of points void PrintParams(); - double FrameNToTime(int64_t frame_number); + double FrameNToTime(int64_t frame_number, double time_scale); BBox InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target); + /// Get and Set JSON methods + std::string Json(); ///< Generate JSON string of this object + Json::Value JsonValue(); ///< Generate Json::Value for this object + void SetJson(const std::string value); ///< Load JSON string into this object + void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object + + void clear(); //clear all fields + + }; } diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 78fd3e079..5205a7460 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -37,7 +37,10 @@ Tracker::Tracker(std::string clipTrackerDataPath) { // Init effect properties init_effect_details(); - +/* + this->trackedData.SetBaseFPS(fps); + this->trackedData.SetOpFPS(fps); +*/ // Tries to load the tracker data from protobuf LoadTrackedData(clipTrackerDataPath); } @@ -47,7 +50,10 @@ Tracker::Tracker() { // Init effect properties init_effect_details(); - +/* + this->trackedData.SetBaseFPS(fps); + this->trackedData.SetOpFPS(fps); +*/ } // Init effect settings @@ -62,6 +68,7 @@ void Tracker::init_effect_details() info.description = "Track the selected bounding box through the video."; info.has_audio = false; info.has_video = true; + } // This method is required for all derived classes of EffectBase, and returns a @@ -76,17 +83,20 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f if(!frame_image.empty()){ // Check if track data exists for the requested frame - if (trackedDataById.find(frame_number) != trackedDataById.end()) { + if (trackedData.Contains(frame_number)) { float fw = frame_image.size().width; float fh = frame_image.size().height; // Draw box on image - EffectFrameData fd = trackedDataById[frame_number]; - cv::Rect2d box((int)(fd.x1*fw), - (int)(fd.y1*fh), - (int)((fd.x2-fd.x1)*fw), - (int)((fd.y2-fd.y1)*fh)); + //EffectFrameData fd = trackedDataById[frame_number]; + + BBox fd = this->trackedData.GetValue(frame_number); + + cv::Rect2d box((int)(fd.cx*fw), + (int)(fd.cy*fh), + (int)(fd.width*fw), + (int)(fd.height*fh)); cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); } } @@ -113,7 +123,9 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ } // Make sure the trackedData is empty - trackedDataById.clear(); + //trackedDataById.clear(); + trackedData.clear(); + // Iterate over all frames of the saved message for (size_t i = 0; i < trackerMessage.frame_size(); i++) { @@ -131,8 +143,10 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ float y2 = box.y2(); // Assign data to tracker map - trackedDataById[id] = EffectFrameData(id, rotation, x1, y1, x2, y2); - } + //trackedDataById[id] = EffectFrameData(id, rotation, x1, y1, x2, y2); + trackedData.AddBox(id, x1, y1, (x2-x1), (y2-y1)); + trackedData.AddRotation(id, rotation); + } // Show the time stamp from the last update in tracker data file if (trackerMessage.has_last_updated()) { @@ -146,15 +160,8 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ } // Get tracker info for the desired frame -EffectFrameData Tracker::GetTrackedData(size_t frameId){ - - // Check if the tracker info for the requested frame exists - if ( trackedDataById.find(frameId) == trackedDataById.end() ) { - return EffectFrameData(); - } else { - return trackedDataById[frameId]; - } - +BBox Tracker::GetTrackedData(size_t frameId){ + return this->trackedData.GetValue(frameId); } // Generate JSON string of this object @@ -171,6 +178,8 @@ Json::Value Tracker::JsonValue() const { Json::Value root = EffectBase::JsonValue(); // get parent properties root["type"] = info.class_name; root["protobuf_data_path"] = protobuf_data_path; + root["BaseFPS"]["num"] = BaseFPS.num; + root["BaseFPS"]["den"] = BaseFPS.den; // return JsonValue return root; @@ -207,6 +216,13 @@ void Tracker::SetJsonValue(const Json::Value root) { protobuf_data_path = ""; } } + + if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject()) { + if (!root["BaseFPS"]["num"].isNull()) + BaseFPS.num = (int) root["BaseFPS"]["num"].asInt(); + if (!root["BaseFPS"]["den"].isNull()) + BaseFPS.den = (int) root["BaseFPS"]["den"].asInt(); + } } // Get all properties for a specific frame diff --git a/src/effects/Tracker.h b/src/effects/Tracker.h index abc816e15..3350428ae 100644 --- a/src/effects/Tracker.h +++ b/src/effects/Tracker.h @@ -42,6 +42,8 @@ #include "../Color.h" #include "../Json.h" #include "../KeyFrame.h" +#include "../KeyFrameBBox.h" +#include "../Clip.h" #include "trackerdata.pb.h" using namespace std; @@ -89,11 +91,13 @@ namespace openshot /// Init effect settings void init_effect_details(); std::string protobuf_data_path; - + Fraction BaseFPS; public: std::map trackedDataById; // Save object tracking box data + KeyFrameBBox trackedData; + /// Blank constructor, useful when using Json to load the effect properties Tracker(std::string clipTrackerDataPath); @@ -116,7 +120,7 @@ namespace openshot bool LoadTrackedData(std::string inputFilePath); // Get tracker info for the desired frame - EffectFrameData GetTrackedData(size_t frameId); + BBox GetTrackedData(size_t frameId); /// Get and Set JSON methods std::string Json() const override; ///< Generate JSON string of this object diff --git a/tests/KeyFrame_Tests.cpp b/tests/KeyFrame_Tests.cpp index b38eb8c9b..13a4c68f7 100644 --- a/tests/KeyFrame_Tests.cpp +++ b/tests/KeyFrame_Tests.cpp @@ -533,4 +533,65 @@ TEST(KeyFrameBBox_GetVal_test) { CHECK_EQUAL(30.0, val.cy); CHECK_EQUAL(130.0,val.width); CHECK_EQUAL(130.0, val.height); -} \ No newline at end of file +} + + +TEST(KeyFrameBBox_GetVal_Interpolation) { + KeyFrameBBox kfb; + + kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0); + kfb.AddBox(11, 20.0, 20.0, 100.0, 100.0); + kfb.AddBox(21, 30.0, 30.0, 100.0, 100.0); + kfb.AddBox(31, 40.0, 40.0, 100.0, 100.0); + + + //kfb.AddDisplacement(1, 20.0, 20.0); + //kfb.AddScale(1, 30, 30); + + BBox val = kfb.GetValue(5); + + CHECK_EQUAL(14.0, val.cx); + CHECK_EQUAL(14.0, val.cy); + CHECK_EQUAL(100.0,val.width); + CHECK_EQUAL(100.0, val.height); + + val = kfb.GetValue(15); + + CHECK_EQUAL(24.0, val.cx); + CHECK_EQUAL(24.0, val.cy); + CHECK_EQUAL(100.0,val.width); + CHECK_EQUAL(100.0, val.height); + + val = kfb.GetValue(25); + + CHECK_EQUAL(34.0, val.cx); + CHECK_EQUAL(34.0, val.cy); + CHECK_EQUAL(100.0,val.width); + CHECK_EQUAL(100.0, val.height); + +} + + +TEST(KeyFrameBBox_Json_set) { + KeyFrameBBox kfb; + + kfb.AddBox(0, 10.0, 10.0, 100.0, 100.0); + kfb.AddBox(10, 20.0, 20.0, 100.0, 100.0); + kfb.AddBox(20, 30.0, 30.0, 100.0, 100.0); + kfb.AddBox(30, 40.0, 40.0, 100.0, 100.0); + + kfb.SetBaseFPS(Fraction(24.0, 1.0)); + kfb.ScalePoints(1.2); + auto data = kfb.Json(); + + KeyFrameBBox from_json; + from_json.SetJson(data); + + BBox val = kfb.GetValue(0); + BBox val_json = from_json.GetValue(0); + + std::cout << from_json.Json() << std::endl; + + CHECK_EQUAL(val.cx, val_json.cx); +} +