From dd9e203695e77d3f9feb5da8bbc6d419856afedb Mon Sep 17 00:00:00 2001 From: Brenno Date: Sat, 28 Nov 2020 18:09:10 -0300 Subject: [PATCH] Added transform handler for Tracker effect It's now possible to adjust the tracked bounding box from the preview window. Some work is needed to make it generic to the ObjectDetection effect and future effects --- src/KeyFrameBBox.cpp | 20 ++++++++--------- src/KeyFrameBBox.h | 34 ++++++++++++++++++---------- src/effects/Tracker.cpp | 48 ++++++++++++++++++++++++++++------------ tests/KeyFrame_Tests.cpp | 16 +++++++------- 4 files changed, 74 insertions(+), 44 deletions(-) diff --git a/src/KeyFrameBBox.cpp b/src/KeyFrameBBox.cpp index 9c7c4ea0b..870506a8e 100644 --- a/src/KeyFrameBBox.cpp +++ b/src/KeyFrameBBox.cpp @@ -164,12 +164,12 @@ void KeyFrameBBox::AddRotation(int64_t _frame_num, double rot){ rotation.AddPoint(time, rot, openshot::InterpolationType::LINEAR); } */ -void KeyFrameBBox::AddBox(int64_t _frame_num , float _cx, float _cy, float _width, float _height){ +void KeyFrameBBox::AddBox(int64_t _frame_num , float _x1, float _y1, float _width, float _height){ if (_frame_num < 0) return; - BBox box = BBox(_cx, _cy, _width, _height); + BBox box = BBox(_x1, _y1, _width, _height); double time = this->FrameNToTime(_frame_num, 1.0); @@ -296,8 +296,8 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number){ if ((it->first == time) || (it == BoxVec.begin())){ BBox res = it->second; - /*res.cx += this->delta_x.GetValue(it->first); - res.cy += this->delta_y.GetValue(it->first); + /*res.x1 += this->delta_x.GetValue(it->first); + res.y1 += this->delta_y.GetValue(it->first); res.height += this->scale_y.GetValue(it->first); res.width += this->scale_x.GetValue(it->first); */ @@ -311,8 +311,8 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number){ /*later add rotation transform to these points*/ /* - res.cx += this->delta_x.GetValue(time); - res.cy += this->delta_y.GetValue(time); + res.x1 += this->delta_x.GetValue(time); + res.y1 += this->delta_y.GetValue(time); res.height += this->scale_y.GetValue(time); res.width += this->scale_x.GetValue(time); */ @@ -323,13 +323,13 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number){ BBox KeyFrameBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target){ - Point p1_left(t1, left.cx, openshot::InterpolationType::LINEAR); - Point p1_right(t2, right.cx, openshot::InterpolationType::LINEAR); + Point p1_left(t1, left.x1, openshot::InterpolationType::LINEAR); + Point p1_right(t2, right.x1, openshot::InterpolationType::LINEAR); Point p1 = InterpolateBetween(p1_left, p1_right, target, 0.01); - Point p2_left(t1, left.cy, openshot::InterpolationType::LINEAR); - Point p2_right(t2, right.cy, openshot::InterpolationType::LINEAR); + Point p2_left(t1, left.y1, openshot::InterpolationType::LINEAR); + Point p2_right(t2, right.y1, openshot::InterpolationType::LINEAR); Point p2 = InterpolateBetween(p2_left, p2_right, target, 0.01); diff --git a/src/KeyFrameBBox.h b/src/KeyFrameBBox.h index 803492862..9828812fd 100644 --- a/src/KeyFrameBBox.h +++ b/src/KeyFrameBBox.h @@ -62,8 +62,8 @@ namespace openshot { struct BBox{ - float cx = -1; - float cy = -1; + float x1 = -1; + float y1 = -1; float width = -1; float height = -1; @@ -72,14 +72,16 @@ namespace openshot { return; } - BBox(float _cx, float _cy, float _width, float _height){ + BBox(float _x1, float _y1, float _width, float _height){ //frame_num = _frame_num; - cx = _cx; - cy = _cy; + x1 = _x1; + y1 = _y1; width = _width; height = _height; } + + std::string Json() const { // Return formatted string return JsonValue().toStyledString(); @@ -90,8 +92,8 @@ namespace openshot { // Create root json object Json::Value root; - root["cx"] = cx; - root["cy"] = cy; + root["x1"] = x1; + root["y1"] = y1; root["height"] = height; root["width"] = width; @@ -119,10 +121,10 @@ namespace openshot { void SetJsonValue(const Json::Value root) { // Set data from Json (if key is found) - if (!root["cx"].isNull()) - cx = root["cx"].asDouble(); - if (!root["cy"].isNull()) - cy = root["cy"].asDouble(); + if (!root["x1"].isNull()) + x1 = root["x1"].asDouble(); + if (!root["y1"].isNull()) + y1 = root["y1"].asDouble(); if (!root["height"].isNull()) height = root["height"].asDouble(); if (!root["width"].isNull()) @@ -143,12 +145,14 @@ namespace openshot { //Keyframe scale_x; //Keyframe scale_y; //Keyframe rotation; + + KeyFrameBBox(); //void AddDisplacement(int64_t _frame_num, double _delta_x, double _delta_y); //void AddScale(int64_t _frame_num, double _delta_x, double _delta_y); - void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height); + void AddBox(int64_t _frame_num, float _x1, float _y1, float _width, float _height); //void AddRotation(int64_t _frame_num, double rot); void SetBaseFPS(Fraction fps); @@ -166,7 +170,13 @@ namespace openshot { //void RemoveScale(int64_t frame_number); //void RemoveRotation(int64_t frame_number); + BBox GetValue(int64_t frame_number) const + { + return const_cast(this)->GetValue(frame_number); + } + BBox GetValue(int64_t frame_number); + /// Print collection of points //void PrintParams(); diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 138ff3ac6..17e234615 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -33,7 +33,7 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Tracker::Tracker(std::string clipTrackerDataPath): delta_x(0.0), delta_y(0.0), scale_x(0.0), scale_y(0.0), rotation(0.0) +Tracker::Tracker(std::string clipTrackerDataPath): delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y(1.0), rotation(0.0) { // Init effect properties init_effect_details(); @@ -42,7 +42,7 @@ Tracker::Tracker(std::string clipTrackerDataPath): delta_x(0.0), delta_y(0.0), s } // Default constructor -Tracker::Tracker(): delta_x(0.0), delta_y(0.0), scale_x(0.0), scale_y(0.0), rotation(0.0) +Tracker::Tracker(): delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y(1.0), rotation(0.0) { // Init effect properties init_effect_details(); @@ -88,16 +88,19 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f double delta_x = this->delta_x.GetValue(frame_number); double delta_y = this->delta_y.GetValue(frame_number); + // convert to [cx, cy, width, height]. Apply scale and translation + BBox fd = this->trackedData.GetValue(frame_number); + float cx = fd.x1 + (fd.width/2) + delta_x; + float cy = fd.y1 + (fd.height/2) + delta_y; + float width = fd.width * scale_x; + float height = fd.height * scale_y; - // Draw box on image - //EffectFrameData fd = trackedDataById[frame_number]; - BBox fd = this->trackedData.GetValue(frame_number); - - cv::Rect2d box((int)( (fd.cx + delta_x) * fw ), - (int)( (fd.cy + delta_y) * fh ), - (int)( (fd.width + scale_x) * fw), - (int)( (fd.height + scale_y) * fh) ); + // Draw box on image + cv::Rect2d box((int)( (cx - (width/2) ) * fw ), + (int)( (cy - (height/2) ) * fh ), + (int)( (width) * fw), + (int)( (height) * fh) ); cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); } } @@ -256,6 +259,7 @@ void Tracker::SetJsonValue(const Json::Value root) { } + // Get all properties for a specific frame std::string Tracker::PropertiesJSON(int64_t requested_frame) const { @@ -269,12 +273,28 @@ std::string Tracker::PropertiesJSON(int64_t requested_frame) const { root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); // Keyframes - root["delta_x"] = add_property_json("Displacement X-axis", delta_x.GetValue(requested_frame), "float", "", &delta_x, -1.0, 1.0, false, requested_frame); - root["delta_y"] = add_property_json("Displacement Y-axis", delta_y.GetValue(requested_frame), "float", "", &delta_y, -1.0, 1.0, false, requested_frame); - root["scale_x"] = add_property_json("Scale (Width)", scale_x.GetValue(requested_frame), "float", "", &scale_x, -1.0, 1.0, false, requested_frame); - root["scale_y"] = add_property_json("Scale (Height)", scale_y.GetValue(requested_frame), "float", "", &scale_y, -1.0, 1.0, false, requested_frame); + float scale_x_value = this->scale_x.GetValue(requested_frame); + float scale_y_value = this->scale_y.GetValue(requested_frame); + float delta_x_value = this->delta_x.GetValue(requested_frame); + float delta_y_value = this->delta_y.GetValue(requested_frame); + root["delta_x"] = add_property_json("Displacement X-axis", this->delta_x.GetValue(requested_frame), "float", "", &delta_x, -1.0, 1.0, false, requested_frame); + root["delta_y"] = add_property_json("Displacement Y-axis", this->delta_y.GetValue(requested_frame), "float", "", &delta_y, -1.0, 1.0, false, requested_frame); + root["scale_x"] = add_property_json("Scale (Width)", this->scale_x.GetValue(requested_frame), "float", "", &scale_x, 0.0, 1.0, false, requested_frame); + root["scale_y"] = add_property_json("Scale (Height)", this->scale_y.GetValue(requested_frame), "float", "", &scale_y, 0.0, 1.0, false, requested_frame); root["rotation"] = add_property_json("Rotation", rotation.GetValue(requested_frame), "float", "", &rotation, 0, 360, false, requested_frame); + // TODO: use p1, p2 convention instead of [x1, y1, width, height] + BBox fd = this->trackedData.GetValue(requested_frame); + float cx = fd.x1 + (fd.width/2) + delta_x_value; + float cy = fd.y1 + (fd.height/2) + delta_y_value; + float width = fd.width * scale_x_value; + float height = fd.height * scale_y_value; + + root["x1"] = add_property_json("X1", cx-(width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); + root["y1"] = add_property_json("Y1", cy-(height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); + root["x2"] = add_property_json("X2", cx+(width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); + root["y2"] = add_property_json("Y2", cy+(height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); + // Return formatted string return root.toStyledString(); } diff --git a/tests/KeyFrame_Tests.cpp b/tests/KeyFrame_Tests.cpp index 56192b1d3..c5d5b6da7 100644 --- a/tests/KeyFrame_Tests.cpp +++ b/tests/KeyFrame_Tests.cpp @@ -529,8 +529,8 @@ TEST(KeyFrameBBox_GetVal_test) { BBox val = kfb.GetValue(1); - CHECK_EQUAL(10.0, val.cx); - CHECK_EQUAL(10.0, val.cy); + CHECK_EQUAL(10.0, val.x1); + CHECK_EQUAL(10.0, val.y1); CHECK_EQUAL(100.0,val.width); CHECK_EQUAL(100.0, val.height); } @@ -550,22 +550,22 @@ TEST(KeyFrameBBox_GetVal_Interpolation) { BBox val = kfb.GetValue(5); - CHECK_EQUAL(14.0, val.cx); - CHECK_EQUAL(14.0, val.cy); + CHECK_EQUAL(14.0, val.x1); + CHECK_EQUAL(14.0, val.y1); CHECK_EQUAL(100.0,val.width); CHECK_EQUAL(100.0, val.height); val = kfb.GetValue(15); - CHECK_EQUAL(24.0, val.cx); - CHECK_EQUAL(24.0, val.cy); + CHECK_EQUAL(24.0, val.x1); + CHECK_EQUAL(24.0, val.y1); CHECK_EQUAL(100.0,val.width); CHECK_EQUAL(100.0, val.height); val = kfb.GetValue(25); - CHECK_EQUAL(34.0, val.cx); - CHECK_EQUAL(34.0, val.cy); + CHECK_EQUAL(34.0, val.x1); + CHECK_EQUAL(34.0, val.y1); CHECK_EQUAL(100.0,val.width); CHECK_EQUAL(100.0, val.height);