From 3af6e1f67202a8450479b56f88439fb799cd0f3d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 22 Feb 2024 16:22:48 -0600 Subject: [PATCH 01/15] Refactor of Tracker effect and TrackedObjectBBox: - Draw Tracker boxes using QPainter and support corner radius (and faster drawing performance by drawing directly on the frame) - Draw child clips with correct aspect ratio - Adding "Yes/No" options for "Visible" and "Draw Box" Tracked Object Box keyframes - default to invisible background - default to 12 corner radius - default to 50% stroke alpha --- src/TrackedObjectBBox.cpp | 17 +-- src/effects/Tracker.cpp | 216 +++++++++++++++++--------------------- src/effects/Tracker.h | 5 +- 3 files changed, 106 insertions(+), 132 deletions(-) diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index 5ca98270f..55400d2d7 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -26,20 +26,19 @@ using namespace openshot; // Default Constructor, delegating TrackedObjectBBox::TrackedObjectBBox() - : TrackedObjectBBox::TrackedObjectBBox(0, 0, 255, 0) {} + : TrackedObjectBBox::TrackedObjectBBox(0, 0, 255, 255) {} // Constructor that takes RGBA values for stroke, and sets the bounding-box // displacement as 0 and the scales as 1 for the first frame TrackedObjectBBox::TrackedObjectBBox(int Red, int Green, int Blue, int Alfa) : delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y(1.0), rotation(0.0), - background_alpha(1.0), background_corner(0), - stroke_width(2) , stroke_alpha(0.0), + background_alpha(0.0), background_corner(12), + stroke_width(2) , stroke_alpha(0.5), stroke(Red, Green, Blue, Alfa), - background(0, 0, 255, 0) + background(0, 0, 255, Alfa) { this->TimeScale = 1.0; - return; } // Add a BBox to the BoxVec map @@ -442,10 +441,12 @@ Json::Value TrackedObjectBBox::PropertiesJSON(int64_t requested_frame) const root["scale_y"] = add_property_json("Scale (Height)", scale_y.GetValue(requested_frame), "float", "", &scale_y, 0.0, 1.0, false, requested_frame); root["rotation"] = add_property_json("Rotation", rotation.GetValue(requested_frame), "float", "", &rotation, 0, 360, false, requested_frame); root["visible"] = add_property_json("Visible", visible.GetValue(requested_frame), "int", "", &visible, 0, 1, false, requested_frame); + root["visible"]["choices"].append(add_property_choice_json("Yes", true, visible.GetValue(requested_frame))); + root["visible"]["choices"].append(add_property_choice_json("No", false, visible.GetValue(requested_frame))); - root["draw_box"] = add_property_json("Draw Box", draw_box.GetValue(requested_frame), "int", "", &draw_box, -1, 1.0, false, requested_frame); - root["draw_box"]["choices"].append(add_property_choice_json("Off", 0, draw_box.GetValue(requested_frame))); - root["draw_box"]["choices"].append(add_property_choice_json("On", 1, draw_box.GetValue(requested_frame))); + root["draw_box"] = add_property_json("Draw Box", draw_box.GetValue(requested_frame), "int", "", &draw_box, 0, 1, false, requested_frame); + root["draw_box"]["choices"].append(add_property_choice_json("Yes", true, draw_box.GetValue(requested_frame))); + root["draw_box"]["choices"].append(add_property_choice_json("No", false, draw_box.GetValue(requested_frame))); root["stroke"] = add_property_json("Border", 0.0, "color", "", NULL, 0, 255, false, requested_frame); root["stroke"]["red"] = add_property_json("Red", stroke.red.GetValue(requested_frame), "float", "", &stroke.red, 0, 255, false, requested_frame); diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 7a0a3df52..094b58d7e 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -13,7 +13,6 @@ #include #include -#include #include #include "effects/Tracker.h" @@ -25,6 +24,8 @@ #include #include +#include +#include #include using namespace std; @@ -83,129 +84,100 @@ void Tracker::init_effect_details() // This method is required for all derived classes of EffectBase, and returns a // modified openshot::Frame object -std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t frame_number) -{ - // Get the frame's image - cv::Mat frame_image = frame->GetImageCV(); - - // Initialize the Qt rectangle that will hold the positions of the bounding-box - QRectF boxRect; - // Initialize the image of the TrackedObject child clip - std::shared_ptr childClipImage = nullptr; - - // Check if frame isn't NULL - if(!frame_image.empty() && - trackedData->Contains(frame_number) && - trackedData->visible.GetValue(frame_number) == 1) - { - // Get the width and height of the image - float fw = frame_image.size().width; - float fh = frame_image.size().height; - - // Get the bounding-box of given frame - BBox fd = trackedData->GetBox(frame_number); - - // Check if track data exists for the requested frame - if (trackedData->draw_box.GetValue(frame_number) == 1) - { - std::vector stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number); - int stroke_width = trackedData->stroke_width.GetValue(frame_number); - float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number); - std::vector bg_rgba = trackedData->background.GetColorRGBA(frame_number); - float bg_alpha = trackedData->background_alpha.GetValue(frame_number); - - // Create a rotated rectangle object that holds the bounding box - cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ), - cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ), - (int) (fd.angle) ); - - DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true); - DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false); - } - - // Get the image of the Tracked Object' child clip - if (trackedData->ChildClipId() != ""){ - // Cast the parent timeline of this effect - Timeline* parentTimeline = static_cast(ParentTimeline()); - if (parentTimeline){ - // Get the Tracked Object's child clip - Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId()); - if (childClip){ - // Get the image of the child clip for this frame - std::shared_ptr childClipFrame = childClip->GetFrame(frame_number); - childClipImage = childClipFrame->GetImage(); - - // Set the Qt rectangle with the bounding-box properties - boxRect.setRect((int)((fd.cx-fd.width/2)*fw), - (int)((fd.cy - fd.height/2)*fh), - (int)(fd.width*fw), - (int)(fd.height*fh) ); - } - } - } - - } - - // Set image with drawn box to frame - // If the input image is NULL or doesn't have tracking data, it's returned as it came - frame->SetImageCV(frame_image); - - // Set the bounding-box image with the Tracked Object's child clip image - if (childClipImage){ - // Get the frame image - QImage frameImage = *(frame->GetImage()); - - // Set a Qt painter to the frame image - QPainter painter(&frameImage); - - // Draw the child clip image inside the bounding-box - painter.drawImage(boxRect, *childClipImage); - - // Set the frame image as the composed image - frame->AddImage(std::make_shared(frameImage)); - } - - return frame; +std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t frame_number) { + // Get the frame's QImage + std::shared_ptr frame_image = frame->GetImage(); + + // Check if frame isn't NULL + if(frame_image && !frame_image->isNull() && + trackedData->Contains(frame_number) && + trackedData->visible.GetValue(frame_number) == 1) { + QPainter painter(frame_image.get()); + + // Get the bounding-box of the given frame + BBox fd = trackedData->GetBox(frame_number); + + // Create a QRectF for the bounding box + QRectF boxRect((fd.cx - fd.width / 2) * frame_image->width(), + (fd.cy - fd.height / 2) * frame_image->height(), + fd.width * frame_image->width(), + fd.height * frame_image->height()); + + // Check if track data exists for the requested frame + if (trackedData->draw_box.GetValue(frame_number) == 1) { + painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform); + + // Get trackedObjectBox keyframes + std::vector stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number); + int stroke_width = trackedData->stroke_width.GetValue(frame_number); + float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number); + std::vector bg_rgba = trackedData->background.GetColorRGBA(frame_number); + float bg_alpha = trackedData->background_alpha.GetValue(frame_number); + float bg_corner = trackedData->background_corner.GetValue(frame_number); + + // Set the pen for the border + QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha)); + pen.setWidth(stroke_width); + painter.setPen(pen); + + // Set the brush for the background + QBrush brush(QColor(bg_rgba[0], bg_rgba[1], bg_rgba[2], 255 * bg_alpha)); + painter.setBrush(brush); + + // Draw the rounded rectangle + painter.drawRoundedRect(boxRect, bg_corner, bg_corner); + } + + // Get the image of the Tracked Object' child clip + if (trackedData->ChildClipId() != ""){ + // Cast the parent timeline of this effect + Timeline* parentTimeline = static_cast(ParentTimeline()); + if (parentTimeline){ + // Get the Tracked Object's child clip + Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId()); + if (childClip){ + // Get the image of the child clip for this frame + std::shared_ptr childClipFrame = childClip->GetFrame(frame_number); + std::shared_ptr childClipImage = childClipFrame->GetImage(); + + // Scale the original bounding box to this image + QRectF scaledRect = scaleAndCenterRect(QRectF(childClipImage->rect()), boxRect); + QImage scaledImage = childClipImage->scaled(scaledRect.size().toSize(), Qt::KeepAspectRatio, Qt::SmoothTransformation); + + // Draw the child clip image inside the bounding-box + painter.drawImage(scaledRect, *childClipImage); + } + } + } + + painter.end(); + } + + // No need to set the image back to the frame, as we directly modified the frame's QImage + return frame; } -void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector color, float alpha, int thickness, bool is_background){ - // Get the bouding box vertices - cv::Point2f vertices2f[4]; - box.points(vertices2f); - - // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed - // select min enclosing rectangle to draw on a small portion of the image - // cv::Rect rect = box.boundingRect(); - // cv::Mat image = frame_image(rect) - - if(is_background){ - cv::Mat overlayFrame; - frame_image.copyTo(overlayFrame); - - // draw bounding box background - cv::Point vertices[4]; - for(int i = 0; i < 4; ++i){ - vertices[i] = vertices2f[i];} - - cv::Rect rect = box.boundingRect(); - cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA); - // add opacity - cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image); - } - else{ - cv::Mat overlayFrame; - frame_image.copyTo(overlayFrame); - - // Draw bounding box - for (int i = 0; i < 4; i++) - { - cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]), - thickness, cv::LINE_AA); - } - - // add opacity - cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image); - } +QRectF Tracker::scaleAndCenterRect(const QRectF& sourceRect, const QRectF& targetRect) { + float sourceAspectRatio = sourceRect.width() / sourceRect.height(); + float targetWidth = targetRect.width(); + float targetHeight = targetRect.height(); + float newWidth, newHeight; + + if (sourceAspectRatio > targetRect.width() / targetRect.height()) { + // Source is wider relative to target, so it's constrained by target's width + newWidth = targetWidth; + newHeight = newWidth / sourceAspectRatio; + } else { + // Source is taller relative to target, so it's constrained by target's height + newHeight = targetHeight; + newWidth = newHeight * sourceAspectRatio; + } + + // Center the new rectangle within the target rectangle + float newX = targetRect.left() + (targetWidth - newWidth) / 2.0; + float newY = targetRect.top() + (targetHeight - newHeight) / 2.0; + + return QRectF(newX, newY, newWidth, newHeight); } // Get the indexes and IDs of all visible objects in the given frame diff --git a/src/effects/Tracker.h b/src/effects/Tracker.h index dc9e718f3..2b6b574e8 100644 --- a/src/effects/Tracker.h +++ b/src/effects/Tracker.h @@ -44,6 +44,9 @@ namespace openshot /// Init effect settings void init_effect_details(); + /// Find a rectangle inside another (centered) + QRectF scaleAndCenterRect(const QRectF& sourceRect, const QRectF& targetRect); + Fraction BaseFPS; double TimeScale; @@ -71,8 +74,6 @@ namespace openshot /// Get the indexes and IDs of all visible objects in the given frame std::string GetVisibleObjects(int64_t frame_number) const override; - void DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector color, float alpha, int thickness, bool is_background); - // Get and Set JSON methods /// Generate JSON string of this object From 98383fdddd2641d74789025242f7785b31976b23 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 22 Feb 2024 20:47:21 -0600 Subject: [PATCH 02/15] Refactor of ObjectDetection effect: - Draw bounding boxes using QPainter and support corner radius (and faster drawing performance by drawing directly on the frame) - Draw child clips with correct aspect ratio - Adding "Yes/No" option for "Draw Text" - default to fully opaque stroke --- src/TrackedObjectBBox.cpp | 2 +- src/effects/ObjectDetection.cpp | 329 ++++++++++++-------------------- src/effects/ObjectDetection.h | 7 - src/effects/Tracker.cpp | 3 - src/effects/Tracker.h | 6 +- 5 files changed, 123 insertions(+), 224 deletions(-) diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index 55400d2d7..8b6381108 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -34,7 +34,7 @@ TrackedObjectBBox::TrackedObjectBBox(int Red, int Green, int Blue, int Alfa) : delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y(1.0), rotation(0.0), background_alpha(0.0), background_corner(12), - stroke_width(2) , stroke_alpha(0.5), + stroke_width(2) , stroke_alpha(0.70), stroke(Red, Green, Blue, Alfa), background(0, 0, 255, Alfa) { diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index e1278b01d..29209f6c4 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -19,6 +19,7 @@ #include "Exceptions.h" #include "Timeline.h" #include "objdetectdata.pb.h" +#include "Tracker.h" #include #include @@ -67,213 +68,120 @@ void ObjectDetection::init_effect_details() // This method is required for all derived classes of EffectBase, and returns a // modified openshot::Frame object -std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, int64_t frame_number) -{ - // Get the frame's image - cv::Mat cv_image = frame->GetImageCV(); - - // Check if frame isn't NULL - if(cv_image.empty()){ - return frame; - } - - // Initialize the Qt rectangle that will hold the positions of the bounding-box - std::vector boxRects; - // Initialize the image of the TrackedObject child clip - std::vector> childClipImages; - - // Check if track data exists for the requested frame - if (detectionsData.find(frame_number) != detectionsData.end()) { - float fw = cv_image.size().width; - float fh = cv_image.size().height; - - DetectionData detections = detectionsData[frame_number]; - for(int i = 0; i 0 && - std::find(display_classes.begin(), display_classes.end(), classNames[detections.classIds.at(i)]) == display_classes.end()){ - continue; - } - - // Get the object id - int objectId = detections.objectIds.at(i); - - // Search for the object in the trackedObjects map - auto trackedObject_it = trackedObjects.find(objectId); - - // Cast the object as TrackedObjectBBox - std::shared_ptr trackedObject = std::static_pointer_cast(trackedObject_it->second); - - // Check if the tracked object has data for this frame - if (trackedObject->Contains(frame_number) && - trackedObject->visible.GetValue(frame_number) == 1) - { - // Get the bounding-box of given frame - BBox trackedBox = trackedObject->GetBox(frame_number); - bool draw_text = !display_box_text.GetValue(frame_number); - std::vector stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number); - int stroke_width = trackedObject->stroke_width.GetValue(frame_number); - float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number); - std::vector bg_rgba = trackedObject->background.GetColorRGBA(frame_number); - float bg_alpha = trackedObject->background_alpha.GetValue(frame_number); - - cv::Rect2d box( - (int)( (trackedBox.cx-trackedBox.width/2)*fw), - (int)( (trackedBox.cy-trackedBox.height/2)*fh), - (int)( trackedBox.width*fw), - (int)( trackedBox.height*fh) - ); - - // If the Draw Box property is off, then make the box invisible - if (trackedObject->draw_box.GetValue(frame_number) == 0) - { - bg_alpha = 1.0; - stroke_alpha = 1.0; - } - - drawPred(detections.classIds.at(i), detections.confidences.at(i), - box, cv_image, detections.objectIds.at(i), bg_rgba, bg_alpha, 1, true, draw_text); - drawPred(detections.classIds.at(i), detections.confidences.at(i), - box, cv_image, detections.objectIds.at(i), stroke_rgba, stroke_alpha, stroke_width, false, draw_text); - - - // Get the Detected Object's child clip - if (trackedObject->ChildClipId() != ""){ - // Cast the parent timeline of this effect - Timeline* parentTimeline = static_cast(ParentTimeline()); - if (parentTimeline){ - // Get the Tracked Object's child clip - Clip* childClip = parentTimeline->GetClip(trackedObject->ChildClipId()); - - if (childClip){ - // Get the image of the child clip for this frame - std::shared_ptr childClipFrame = childClip->GetFrame(frame_number); - childClipImages.push_back(childClipFrame->GetImage()); - - // Set the Qt rectangle with the bounding-box properties - QRectF boxRect; - boxRect.setRect((int)((trackedBox.cx-trackedBox.width/2)*fw), - (int)((trackedBox.cy - trackedBox.height/2)*fh), - (int)(trackedBox.width*fw), - (int)(trackedBox.height*fh)); - boxRects.push_back(boxRect); - } - } - } - } - } - } - - // Update Qt image with new Opencv frame - frame->SetImageCV(cv_image); - - // Set the bounding-box image with the Tracked Object's child clip image - if(boxRects.size() > 0){ - // Get the frame image - QImage frameImage = *(frame->GetImage()); - for(int i; i < boxRects.size();i++){ - // Set a Qt painter to the frame image - QPainter painter(&frameImage); - // Draw the child clip image inside the bounding-box - painter.drawImage(boxRects[i], *childClipImages[i]); - } - // Set the frame image as the composed image - frame->AddImage(std::make_shared(frameImage)); - } - - return frame; -} - -void ObjectDetection::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector color, float alpha, - int thickness, bool is_background){ - // Get the bouding box vertices - cv::Point2f vertices2f[4]; - box.points(vertices2f); - - // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed - // select min enclosing rectangle to draw on a small portion of the image - // cv::Rect rect = box.boundingRect(); - // cv::Mat image = frame_image(rect) - - if(is_background){ - cv::Mat overlayFrame; - frame_image.copyTo(overlayFrame); - - // draw bounding box background - cv::Point vertices[4]; - for(int i = 0; i < 4; ++i){ - vertices[i] = vertices2f[i];} - - cv::Rect rect = box.boundingRect(); - cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA); - // add opacity - cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image); - } - else{ - cv::Mat overlayFrame; - frame_image.copyTo(overlayFrame); - - // Draw bounding box - for (int i = 0; i < 4; i++) - { - cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]), - thickness, cv::LINE_AA); - } - - // add opacity - cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image); - } -} - -void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame, int objectNumber, std::vector color, - float alpha, int thickness, bool is_background, bool display_text) -{ - - if(is_background){ - cv::Mat overlayFrame; - frame.copyTo(overlayFrame); - - //Draw a rectangle displaying the bounding box - cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), cv::FILLED); - - // add opacity - cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame); - } - else{ - cv::Mat overlayFrame; - frame.copyTo(overlayFrame); - - //Draw a rectangle displaying the bounding box - cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), thickness); - - if(display_text){ - //Get the label for the class name and its confidence - std::string label = cv::format("%.2f", conf); - if (!classNames.empty()) - { - CV_Assert(classId < (int)classNames.size()); - label = classNames[classId] + ":" + label; - } - - //Display the label at the top of the bounding box - int baseLine; - cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - double left = box.x; - double top = std::max((int)box.y, labelSize.height); - - cv::rectangle(overlayFrame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine), - cv::Scalar(color[2],color[1],color[0]), cv::FILLED); - putText(overlayFrame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1); - } - // add opacity - cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame); - } +std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, int64_t frame_number) { + // Get the frame's QImage + std::shared_ptr frame_image = frame->GetImage(); + + // Check if frame isn't NULL + if(!frame_image || frame_image->isNull()) { + return frame; + } + + QPainter painter(frame_image.get()); + painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform); + + if (detectionsData.find(frame_number) != detectionsData.end()) { + float fw = frame_image->width(); + float fh = frame_image->height(); + + DetectionData detections = detectionsData[frame_number]; + for (int i = 0; i < detections.boxes.size(); i++) { + if (detections.confidences.at(i) < confidence_threshold || + (!display_classes.empty() && + std::find(display_classes.begin(), display_classes.end(), classNames[detections.classIds.at(i)]) == display_classes.end())) { + continue; + } + + int objectId = detections.objectIds.at(i); + auto trackedObject_it = trackedObjects.find(objectId); + + if (trackedObject_it != trackedObjects.end()) { + std::shared_ptr trackedObject = std::static_pointer_cast(trackedObject_it->second); + + if (trackedObject->Contains(frame_number) && trackedObject->visible.GetValue(frame_number) == 1) { + BBox trackedBox = trackedObject->GetBox(frame_number); + + QRectF boxRect((trackedBox.cx - trackedBox.width / 2) * fw, + (trackedBox.cy - trackedBox.height / 2) * fh, + trackedBox.width * fw, + trackedBox.height * fh); + + if (trackedObject->draw_box.GetValue(frame_number) == 1) { + // Draw bounding box + bool display_text = !display_box_text.GetValue(frame_number); + std::vector stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number); + std::vector bg_rgba = trackedObject->background.GetColorRGBA(frame_number); + int stroke_width = trackedObject->stroke_width.GetValue(frame_number); + float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number); + float bg_alpha = trackedObject->background_alpha.GetValue(frame_number); + float bg_corner = trackedObject->background_corner.GetValue(frame_number); + + // Set the pen for the border + QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha)); + pen.setWidth(stroke_width); + painter.setPen(pen); + + // Set the brush for the background + QBrush brush(QColor(bg_rgba[0], bg_rgba[1], bg_rgba[2], 255 * bg_alpha)); + painter.setBrush(brush); + + // Draw the rounded rectangle + painter.drawRoundedRect(boxRect, bg_corner, bg_corner); + + if(display_text) { + // Draw text label above bounding box + // Get the confidence and classId for the current detection + float conf = detections.confidences.at(i); + int classId = detections.classIds.at(i); + + // Get the label for the class name and its confidence + QString label = QString::number(conf, 'f', 2); // Format confidence to two decimal places + if (!classNames.empty()) { + label = QString::fromStdString(classNames[classId]) + ":" + label; + } + + // Set up the painter, font, and pen + QFont font; + font.setPixelSize(14); + painter.setFont(font); + + // Calculate the size of the text + QFontMetrics fontMetrics(font); + QSize labelSize = fontMetrics.size(Qt::TextSingleLine, label); + + // Define the top left point of the rectangle + double left = boxRect.center().x() - (labelSize.width() / 2.0); + double top = std::max(static_cast(boxRect.top()), labelSize.height()) - 4.0; + + // Draw the text + painter.drawText(QPointF(left, top), label); + } + } + + // Get the image of the Tracked Object' child clip + if (trackedObject->ChildClipId() != "") { + Timeline* parentTimeline = static_cast(ParentTimeline()); + if (parentTimeline) { + Clip* childClip = parentTimeline->GetClip(trackedObject->ChildClipId()); + if (childClip) { + std::shared_ptr childClipFrame = childClip->GetFrame(frame_number); + std::shared_ptr childClipImage = childClipFrame->GetImage(); + + // Scale the original bounding box to this image + QRectF scaledRect = Tracker::scaleAndCenterRect(QRectF(childClipImage->rect()), boxRect); + painter.drawImage(scaledRect, *childClipImage); + } + } + } + } + } + } + } + + painter.end(); + + // The frame's QImage has been modified in place, so we just return the original frame + return frame; } // Load protobuf data file @@ -348,6 +256,7 @@ bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){ { // There is no tracked object with that id, so insert a new one TrackedObjectBBox trackedObj((int)classesColor[classId](0), (int)classesColor[classId](1), (int)classesColor[classId](2), (int)0); + trackedObj.stroke_alpha = Keyframe(1.0); trackedObj.AddBox(id, x+(w/2), y+(h/2), w, h, 0.0); std::shared_ptr trackedObjPtr = std::make_shared(trackedObj); @@ -551,9 +460,9 @@ std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const { root["confidence_threshold"] = add_property_json("Confidence Theshold", confidence_threshold, "float", "", NULL, 0, 1, false, requested_frame); root["class_filter"] = add_property_json("Class Filter", 0.0, "string", class_filter, NULL, -1, -1, false, requested_frame); - root["display_box_text"] = add_property_json("Draw Box Text", display_box_text.GetValue(requested_frame), "int", "", &display_box_text, 0, 1.0, false, requested_frame); - root["display_box_text"]["choices"].append(add_property_choice_json("Off", 1, display_box_text.GetValue(requested_frame))); - root["display_box_text"]["choices"].append(add_property_choice_json("On", 0, display_box_text.GetValue(requested_frame))); + root["display_box_text"] = add_property_json("Draw Box Text", display_box_text.GetValue(requested_frame), "int", "", &display_box_text, 0, 1, false, requested_frame); + root["display_box_text"]["choices"].append(add_property_choice_json("Yes", true, display_box_text.GetValue(requested_frame))); + root["display_box_text"]["choices"].append(add_property_choice_json("No", false, display_box_text.GetValue(requested_frame))); // Return formatted string return root.toStyledString(); diff --git a/src/effects/ObjectDetection.h b/src/effects/ObjectDetection.h index d45ab4c67..42f278d10 100644 --- a/src/effects/ObjectDetection.h +++ b/src/effects/ObjectDetection.h @@ -60,7 +60,6 @@ namespace openshot std::string protobuf_data_path; std::map detectionsData; std::vector classNames; - std::vector classesColor; /// Draw class name and confidence score on top of the bounding box @@ -73,12 +72,6 @@ namespace openshot /// Init effect settings void init_effect_details(); - /// Draw bounding box with class and score text - void drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame, int objectNumber, std::vector color, float alpha, - int thickness, bool is_background, bool draw_text); - /// Draw rotated rectangle with alpha channel - void DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector color, float alpha, int thickness, bool is_background); - public: /// Index of the Tracked Object that was selected to modify it's properties diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 094b58d7e..508107a1d 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -142,9 +142,6 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f // Scale the original bounding box to this image QRectF scaledRect = scaleAndCenterRect(QRectF(childClipImage->rect()), boxRect); - QImage scaledImage = childClipImage->scaled(scaledRect.size().toSize(), Qt::KeepAspectRatio, Qt::SmoothTransformation); - - // Draw the child clip image inside the bounding-box painter.drawImage(scaledRect, *childClipImage); } } diff --git a/src/effects/Tracker.h b/src/effects/Tracker.h index 2b6b574e8..c6955a840 100644 --- a/src/effects/Tracker.h +++ b/src/effects/Tracker.h @@ -44,9 +44,6 @@ namespace openshot /// Init effect settings void init_effect_details(); - /// Find a rectangle inside another (centered) - QRectF scaleAndCenterRect(const QRectF& sourceRect, const QRectF& targetRect); - Fraction BaseFPS; double TimeScale; @@ -74,6 +71,9 @@ namespace openshot /// Get the indexes and IDs of all visible objects in the given frame std::string GetVisibleObjects(int64_t frame_number) const override; + /// Find a rectangle inside another (centered) + static QRectF scaleAndCenterRect(const QRectF& sourceRect, const QRectF& targetRect); + // Get and Set JSON methods /// Generate JSON string of this object From 4594cf85b62b8964c8a57e2720803f6565f36bf8 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 22 Feb 2024 21:17:24 -0600 Subject: [PATCH 03/15] Fix Tracker/ObjectDetector unit tests, due to new default values --- src/TrackedObjectBBox.cpp | 2 +- tests/KeyFrame.cpp | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index 8b6381108..0d79a2536 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -34,7 +34,7 @@ TrackedObjectBBox::TrackedObjectBBox(int Red, int Green, int Blue, int Alfa) : delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y(1.0), rotation(0.0), background_alpha(0.0), background_corner(12), - stroke_width(2) , stroke_alpha(0.70), + stroke_width(2) , stroke_alpha(0.7), stroke(Red, Green, Blue, Alfa), background(0, 0, 255, Alfa) { diff --git a/tests/KeyFrame.cpp b/tests/KeyFrame.cpp index a7559103f..49a4d2b20 100644 --- a/tests/KeyFrame.cpp +++ b/tests/KeyFrame.cpp @@ -583,10 +583,10 @@ TEST_CASE( "TrackedObjectBBox init", "[libopenshot][keyframe]" ) CHECK(kfb.rotation.GetInt(1) == 0); CHECK(kfb.stroke_width.GetInt(1) == 2); - CHECK(kfb.stroke_alpha.GetInt(1) == 0); + CHECK(kfb.stroke_alpha.GetValue(1) == Approx(0.7f).margin(0.0001)); - CHECK(kfb.background_alpha .GetInt(1)== 1); - CHECK(kfb.background_corner.GetInt(1) == 0); + CHECK(kfb.background_alpha .GetInt(1) == 0); + CHECK(kfb.background_corner.GetInt(1) == 12); CHECK(kfb.stroke.red.GetInt(1) == 62); CHECK(kfb.stroke.green.GetInt(1) == 143); @@ -596,8 +596,7 @@ TEST_CASE( "TrackedObjectBBox init", "[libopenshot][keyframe]" ) CHECK(kfb.background.red.GetInt(1) == 0); CHECK(kfb.background.green.GetInt(1) == 0); CHECK(kfb.background.blue.GetInt(1) == 255); - CHECK(kfb.background.alpha.GetInt(1) == 0); - + CHECK(kfb.background.alpha.GetInt(1) == 212); } TEST_CASE( "TrackedObjectBBox AddBox and RemoveBox", "[libopenshot][keyframe]" ) From 04ec0232cb59ab7840f78f8c0d7cc6b7d5e2c17b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 24 Feb 2024 16:38:07 -0600 Subject: [PATCH 04/15] Make x1,x2,y1,y2 properties read only in properties window --- src/TrackedObjectBBox.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index 0d79a2536..03087814d 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -429,10 +429,10 @@ Json::Value TrackedObjectBBox::PropertiesJSON(int64_t requested_frame) const root["child_clip_id"] = add_property_json("Child Clip ID", 0.0, "string", ChildClipId(), NULL, -1, -1, false, requested_frame); // Add the data of given frame bounding-box to the JSON object - root["x1"] = add_property_json("X1", box.cx-(box.width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); - root["y1"] = add_property_json("Y1", box.cy-(box.height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); - root["x2"] = add_property_json("X2", box.cx+(box.width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); - root["y2"] = add_property_json("Y2", box.cy+(box.height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame); + root["x1"] = add_property_json("X1", box.cx-(box.width/2), "float", "", NULL, 0.0, 1.0, true, requested_frame); + root["y1"] = add_property_json("Y1", box.cy-(box.height/2), "float", "", NULL, 0.0, 1.0, true, requested_frame); + root["x2"] = add_property_json("X2", box.cx+(box.width/2), "float", "", NULL, 0.0, 1.0, true, requested_frame); + root["y2"] = add_property_json("Y2", box.cy+(box.height/2), "float", "", NULL, 0.0, 1.0, true, requested_frame); // Add the bounding-box Keyframes to the JSON object root["delta_x"] = add_property_json("Displacement X-axis", delta_x.GetValue(requested_frame), "float", "", &delta_x, -1.0, 1.0, false, requested_frame); From a1f67a971694899663c9081bab741c27c8372320 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 25 Feb 2024 23:37:19 -0600 Subject: [PATCH 05/15] Fix class_filter to allow for clearing, and trimming extra spaces --- src/effects/ObjectDetection.cpp | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 29209f6c4..8744d9ae6 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -19,11 +19,12 @@ #include "Exceptions.h" #include "Timeline.h" #include "objdetectdata.pb.h" -#include "Tracker.h" #include #include #include +#include +#include using namespace std; using namespace openshot; @@ -407,18 +408,24 @@ void ObjectDetection::SetJsonValue(const Json::Value root) { if (!root["display_box_text"].isNull()) display_box_text.SetJsonValue(root["display_box_text"]); - if (!root["class_filter"].isNull()){ - class_filter = root["class_filter"].asString(); - std::stringstream ss(class_filter); - display_classes.clear(); - while( ss.good() ) - { - // Parse comma separated string - std::string substr; - std::getline( ss, substr, ',' ); - display_classes.push_back( substr ); - } - } + if (!root["class_filter"].isNull()) { + class_filter = root["class_filter"].asString(); + + // Convert the class_filter to a QString + QString qClassFilter = QString::fromStdString(root["class_filter"].asString()); + + // Split the QString by commas and automatically trim each resulting string + QStringList classList = qClassFilter.split(',', QString::SkipEmptyParts); + display_classes.clear(); + + // Iterate over the QStringList and add each trimmed, non-empty string + for (const QString &classItem : classList) { + QString trimmedItem = classItem.trimmed(); + if (!trimmedItem.isEmpty()) { + display_classes.push_back(trimmedItem.toStdString()); + } + } + } if (!root["objects"].isNull()){ for (auto const& trackedObject : trackedObjects){ From 15be792564feabeabab44580b86a4ba743a16534 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 25 Feb 2024 23:38:57 -0600 Subject: [PATCH 06/15] Refactor of clip caching, to prevent caching flattened images with previous layers, replace std::shared_ptr with QSize for a few arguments to make it much more clear what is happening. --- src/Clip.cpp | 108 +++++++++++++++++++++++++-------------------------- src/Clip.h | 10 ++--- 2 files changed, 57 insertions(+), 61 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 1cced73af..4da0b8562 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -414,47 +414,49 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun // Check cache frame = final_cache.GetFrame(clip_frame_number); - if (frame) { - // Debug output - ZmqLogger::Instance()->AppendDebugMethod( - "Clip::GetFrame (Cached frame found)", - "requested_frame", clip_frame_number); - - // Return cached frame - return frame; - } - - // Generate clip frame - frame = GetOrCreateFrame(clip_frame_number); - - if (!background_frame) { - // Create missing background_frame w/ transparent color (if needed) - background_frame = std::make_shared(clip_frame_number, frame->GetWidth(), frame->GetHeight(), - "#00000000", frame->GetAudioSamplesCount(), - frame->GetAudioChannelsCount()); - } - - // Get time mapped frame object (used to increase speed, change direction, etc...) - apply_timemapping(frame); - - // Apply waveform image (if any) - apply_waveform(frame, background_frame); - - // Apply effects BEFORE applying keyframes (if any local or global effects are used) - apply_effects(frame, background_frame, options, true); - - // Apply keyframe / transforms to current clip image - apply_keyframes(frame, background_frame); - - // Apply effects AFTER applying keyframes (if any local or global effects are used) - apply_effects(frame, background_frame, options, false); + if (!frame) { + // Generate clip frame + frame = GetOrCreateFrame(clip_frame_number); + + // Get frame size and frame # + int64_t timeline_frame_number = clip_frame_number; + QSize timeline_size(frame->GetWidth(), frame->GetHeight()); + if (background_frame) { + // If a background frame is provided, use it instead + timeline_frame_number = background_frame->number; + timeline_size.setWidth(background_frame->GetWidth()); + timeline_size.setHeight(background_frame->GetHeight()); + } + + // Get time mapped frame object (used to increase speed, change direction, etc...) + apply_timemapping(frame); + + // Apply waveform image (if any) + apply_waveform(frame, timeline_size); + + // Apply effects BEFORE applying keyframes (if any local or global effects are used) + apply_effects(frame, timeline_frame_number, options, true); + + // Apply keyframe / transforms to current clip image + apply_keyframes(frame, timeline_size); + + // Apply effects AFTER applying keyframes (if any local or global effects are used) + apply_effects(frame, timeline_frame_number, options, false); + + // Add final frame to cache (before flattening into background_frame) + final_cache.Add(frame); + } + + if (!background_frame) { + // Create missing background_frame w/ transparent color (if needed) + background_frame = std::make_shared(frame->number, frame->GetWidth(), frame->GetHeight(), + "#00000000", frame->GetAudioSamplesCount(), + frame->GetAudioChannelsCount()); + } // Apply background canvas (i.e. flatten this image onto previous layer image) apply_background(frame, background_frame); - // Add final frame to cache - final_cache.Add(frame); - // Return processed 'frame' return frame; } @@ -1221,7 +1223,7 @@ void Clip::apply_background(std::shared_ptr frame, std::shared_ } // Apply effects to the source frame (if any) -void Clip::apply_effects(std::shared_ptr frame, std::shared_ptr background_frame, TimelineInfoStruct* options, bool before_keyframes) +void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes) { for (auto effect : effects) { @@ -1237,18 +1239,18 @@ void Clip::apply_effects(std::shared_ptr frame, std::shared_ptr ba // Apply global timeline effects (i.e. transitions & masks... if any) Timeline* timeline_instance = static_cast(timeline); options->is_before_clip_keyframes = before_keyframes; - timeline_instance->apply_effects(frame, background_frame->number, Layer(), options); + timeline_instance->apply_effects(frame, timeline_frame_number, Layer(), options); } } // Compare 2 floating point numbers for equality -bool Clip::isEqual(double a, double b) +bool Clip::isNear(double a, double b) { return fabs(a - b) < 0.000001; } // Apply keyframes to the source frame (if any) -void Clip::apply_keyframes(std::shared_ptr frame, std::shared_ptr background_frame) { +void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { // Skip out if video was disabled or only an audio frame (no visualisation in use) if (!frame->has_image_data) { // Skip the rest of the image processing for performance reasons @@ -1257,8 +1259,8 @@ void Clip::apply_keyframes(std::shared_ptr frame, std::shared_ptr // Get image from clip, and create transparent background image std::shared_ptr source_image = frame->GetImage(); - std::shared_ptr background_canvas = std::make_shared(background_frame->GetImage()->width(), - background_frame->GetImage()->height(), + std::shared_ptr background_canvas = std::make_shared(timeline_size.width(), + timeline_size.height(), QImage::Format_RGBA8888_Premultiplied); background_canvas->fill(QColor(Qt::transparent)); @@ -1312,7 +1314,7 @@ void Clip::apply_keyframes(std::shared_ptr frame, std::shared_ptr } // Apply apply_waveform image to the source frame (if any) -void Clip::apply_waveform(std::shared_ptr frame, std::shared_ptr background_frame) { +void Clip::apply_waveform(std::shared_ptr frame, QSize timeline_size) { if (!Waveform()) { // Exit if no waveform is needed @@ -1321,15 +1323,14 @@ void Clip::apply_waveform(std::shared_ptr frame, std::shared_ptr b // Get image from clip std::shared_ptr source_image = frame->GetImage(); - std::shared_ptr background_canvas = background_frame->GetImage(); // Debug output ZmqLogger::Instance()->AppendDebugMethod( "Clip::apply_waveform (Generate Waveform Image)", "frame->number", frame->number, "Waveform()", Waveform(), - "background_canvas->width()", background_canvas->width(), - "background_canvas->height()", background_canvas->height()); + "width", timeline_size.width(), + "height", timeline_size.height()); // Get the color of the waveform int red = wave_color.red.GetInt(frame->number); @@ -1338,7 +1339,7 @@ void Clip::apply_waveform(std::shared_ptr frame, std::shared_ptr b int alpha = wave_color.alpha.GetInt(frame->number); // Generate Waveform Dynamically (the size of the timeline) - source_image = frame->GetWaveform(background_canvas->width(), background_canvas->height(), red, green, blue, alpha); + source_image = frame->GetWaveform(timeline_size.width(), timeline_size.height(), red, green, blue, alpha); frame->AddImage(source_image); } @@ -1377,11 +1378,6 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig /* RESIZE SOURCE IMAGE - based on scale type */ QSize source_size = source_image->size(); - // Apply stretch scale to correctly fit the bounding-box - if (parentTrackedObject){ - scale = SCALE_STRETCH; - } - switch (scale) { case (SCALE_FIT): { @@ -1578,11 +1574,11 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig "r", r, "sx", sx, "sy", sy); - if (!isEqual(x, 0) || !isEqual(y, 0)) { + if (!isNear(x, 0) || !isNear(y, 0)) { // TRANSLATE/MOVE CLIP transform.translate(x, y); } - if (!isEqual(r, 0) || !isEqual(shear_x_value, 0) || !isEqual(shear_y_value, 0)) { + if (!isNear(r, 0) || !isNear(shear_x_value, 0) || !isNear(shear_y_value, 0)) { // ROTATE CLIP (around origin_x, origin_y) float origin_x_offset = (scaled_source_width * origin_x_value); float origin_y_offset = (scaled_source_height * origin_y_value); @@ -1594,7 +1590,7 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig // SCALE CLIP (if needed) float source_width_scale = (float(source_size.width()) / float(source_image->width())) * sx; float source_height_scale = (float(source_size.height()) / float(source_image->height())) * sy; - if (!isEqual(source_width_scale, 1.0) || !isEqual(source_height_scale, 1.0)) { + if (!isNear(source_width_scale, 1.0) || !isNear(source_height_scale, 1.0)) { transform.scale(source_width_scale, source_height_scale); } diff --git a/src/Clip.h b/src/Clip.h index 72128e07b..7bae35ccb 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -131,13 +131,13 @@ namespace openshot { void apply_background(std::shared_ptr frame, std::shared_ptr background_frame); /// Apply effects to the source frame (if any) - void apply_effects(std::shared_ptr frame, std::shared_ptr background_frame, TimelineInfoStruct* options, bool before_keyframes); + void apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes); /// Apply keyframes to an openshot::Frame and use an existing background frame (if any) - void apply_keyframes(std::shared_ptr frame, std::shared_ptr background_frame); + void apply_keyframes(std::shared_ptr frame, QSize timeline_size); /// Apply waveform image to an openshot::Frame and use an existing background frame (if any) - void apply_waveform(std::shared_ptr frame, std::shared_ptr background_frame); + void apply_waveform(std::shared_ptr frame, QSize timeline_size); /// Adjust frame number for Clip position and start (which can result in a different number) int64_t adjust_timeline_framenumber(int64_t clip_frame_number); @@ -154,8 +154,8 @@ namespace openshot { /// Adjust the audio and image of a time mapped frame void apply_timemapping(std::shared_ptr frame); - /// Compare 2 floating point numbers - bool isEqual(double a, double b); + /// Compare 2 floating point numbers and return true if they are extremely close + bool isNear(double a, double b); /// Sort effects by order void sort_effects(); From 8f3c324b0b479dc9bd21c8e4eee6c89e74d0642e Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 26 Feb 2024 10:47:38 -0600 Subject: [PATCH 07/15] Set range of background corner radius to 150.0 (which is essentially a circle in my testing). Also, ignoring case of class_filter on Object Detection effect. --- src/TrackedObjectBBox.cpp | 2 +- src/effects/ObjectDetection.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index 03087814d..960939b2b 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -456,7 +456,7 @@ Json::Value TrackedObjectBBox::PropertiesJSON(int64_t requested_frame) const root["stroke_alpha"] = add_property_json("Stroke alpha", stroke_alpha.GetValue(requested_frame), "float", "", &stroke_alpha, 0.0, 1.0, false, requested_frame); root["background_alpha"] = add_property_json("Background Alpha", background_alpha.GetValue(requested_frame), "float", "", &background_alpha, 0.0, 1.0, false, requested_frame); - root["background_corner"] = add_property_json("Background Corner Radius", background_corner.GetValue(requested_frame), "int", "", &background_corner, 0.0, 60.0, false, requested_frame); + root["background_corner"] = add_property_json("Background Corner Radius", background_corner.GetValue(requested_frame), "int", "", &background_corner, 0.0, 150.0, false, requested_frame); root["background"] = add_property_json("Background", 0.0, "color", "", NULL, 0, 255, false, requested_frame); root["background"]["red"] = add_property_json("Red", background.red.GetValue(requested_frame), "float", "", &background.red, 0, 255, false, requested_frame); diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 8744d9ae6..8ef3a1830 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -420,7 +420,7 @@ void ObjectDetection::SetJsonValue(const Json::Value root) { // Iterate over the QStringList and add each trimmed, non-empty string for (const QString &classItem : classList) { - QString trimmedItem = classItem.trimmed(); + QString trimmedItem = classItem.trimmed().toLower(); if (!trimmedItem.isEmpty()) { display_classes.push_back(trimmedItem.toStdString()); } From f15c91db9009bd02667ce611edfaeadea29f7b1b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 26 Feb 2024 16:06:36 -0600 Subject: [PATCH 08/15] Change Object Detector to display the `class name: object id`, instead of the confidence score. This is way more useful, so you can find the object in the properties menu. Also, output the visible class names as well, so the property editor can display them in a context menu. --- src/effects/ObjectDetection.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 8ef3a1830..be6e96ec2 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -132,11 +132,10 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i if(display_text) { // Draw text label above bounding box // Get the confidence and classId for the current detection - float conf = detections.confidences.at(i); int classId = detections.classIds.at(i); // Get the label for the class name and its confidence - QString label = QString::number(conf, 'f', 2); // Format confidence to two decimal places + QString label = QString::number(objectId); if (!classNames.empty()) { label = QString::fromStdString(classNames[classId]) + ":" + label; } @@ -297,6 +296,7 @@ std::string ObjectDetection::GetVisibleObjects(int64_t frame_number) const{ Json::Value root; root["visible_objects_index"] = Json::Value(Json::arrayValue); root["visible_objects_id"] = Json::Value(Json::arrayValue); + root["visible_class_names"] = Json::Value(Json::arrayValue); // Check if track data exists for the requested frame if (detectionsData.find(frame_number) == detectionsData.end()){ @@ -311,11 +311,21 @@ std::string ObjectDetection::GetVisibleObjects(int64_t frame_number) const{ continue; } - // Just display selected classes - if( display_classes.size() > 0 && - std::find(display_classes.begin(), display_classes.end(), classNames[detections.classIds.at(i)]) == display_classes.end()){ - continue; - } + // Get class name of tracked object + auto className = classNames[detections.classIds.at(i)]; + + // If display_classes is not empty, check if className is in it + if (!display_classes.empty()) { + auto it = std::find(display_classes.begin(), display_classes.end(), className); + if (it == display_classes.end()) { + // If not in display_classes, skip this detection + continue; + } + root["visible_class_names"].append(className); + } else { + // include all class names + root["visible_class_names"].append(className); + } int objectId = detections.objectIds.at(i); // Search for the object in the trackedObjects map From 07e4458c03141e100500c8ef45e5d799fdaa0195 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 26 Feb 2024 16:45:30 -0600 Subject: [PATCH 09/15] Removing "ChildClipId" property, no longer used for attaching clips to tracker or object detection effects: Use clip->Parent to attach clips to tracked objects now. --- src/TrackedObjectBBox.cpp | 9 --------- src/TrackedObjectBase.cpp | 2 +- src/TrackedObjectBase.h | 5 ----- src/effects/ObjectDetection.cpp | 16 ---------------- src/effects/Tracker.cpp | 19 ------------------- 5 files changed, 1 insertion(+), 50 deletions(-) diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index 960939b2b..d1a1b5836 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -314,7 +314,6 @@ Json::Value TrackedObjectBBox::JsonValue() const root["BaseFPS"]["num"] = BaseFps.num; root["BaseFPS"]["den"] = BaseFps.den; root["TimeScale"] = TimeScale; - root["child_clip_id"] = ChildClipId(); // Keyframe's properties root["delta_x"] = delta_x.JsonValue(); @@ -379,11 +378,6 @@ void TrackedObjectBBox::SetJsonValue(const Json::Value root) if (!root["protobuf_data_path"].isNull()) protobufDataPath = root["protobuf_data_path"].asString(); - // Set the id of the child clip - if (!root["child_clip_id"].isNull() && root["child_clip_id"].asString() != Id()){ - ChildClipId(root["child_clip_id"].asString()); - } - // Set the Keyframes by the given JSON object if (!root["delta_x"].isNull()) delta_x.SetJsonValue(root["delta_x"]); @@ -425,9 +419,6 @@ Json::Value TrackedObjectBBox::PropertiesJSON(int64_t requested_frame) const // Add the ID of this object to the JSON object root["box_id"] = add_property_json("Box ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame); - // Add the ID of this object's child clip to the JSON object - root["child_clip_id"] = add_property_json("Child Clip ID", 0.0, "string", ChildClipId(), NULL, -1, -1, false, requested_frame); - // Add the data of given frame bounding-box to the JSON object root["x1"] = add_property_json("X1", box.cx-(box.width/2), "float", "", NULL, 0.0, 1.0, true, requested_frame); root["y1"] = add_property_json("Y1", box.cy-(box.height/2), "float", "", NULL, 0.0, 1.0, true, requested_frame); diff --git a/src/TrackedObjectBase.cpp b/src/TrackedObjectBase.cpp index f268ab99a..0fa4a05d5 100644 --- a/src/TrackedObjectBase.cpp +++ b/src/TrackedObjectBase.cpp @@ -23,7 +23,7 @@ namespace openshot // Constructor TrackedObjectBase::TrackedObjectBase(std::string _id) - : visible(1.0), draw_box(1), id(_id), childClipId("") {} + : visible(1.0), draw_box(1), id(_id) {} Json::Value TrackedObjectBase::add_property_choice_json( std::string name, int value, int selected_value) const diff --git a/src/TrackedObjectBase.h b/src/TrackedObjectBase.h index 657def16b..faac178aa 100644 --- a/src/TrackedObjectBase.h +++ b/src/TrackedObjectBase.h @@ -35,8 +35,6 @@ namespace openshot { class TrackedObjectBase { protected: std::string id; - std::string childClipId; - ClipBase* parentClip; public: @@ -60,9 +58,6 @@ namespace openshot { /// Get and set the parentClip of this object ClipBase* ParentClip() const { return parentClip; } void ParentClip(ClipBase* clip) { parentClip = clip; } - /// Get and set the Id of the childClip of this object - std::string ChildClipId() const { return childClipId; }; - void ChildClipId(std::string _childClipId) { childClipId = _childClipId; }; /// Check if there is data for the exact frame number virtual bool ExactlyContains(int64_t frame_number) const { return {}; }; diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index be6e96ec2..9c0d65b79 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -157,22 +157,6 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i painter.drawText(QPointF(left, top), label); } } - - // Get the image of the Tracked Object' child clip - if (trackedObject->ChildClipId() != "") { - Timeline* parentTimeline = static_cast(ParentTimeline()); - if (parentTimeline) { - Clip* childClip = parentTimeline->GetClip(trackedObject->ChildClipId()); - if (childClip) { - std::shared_ptr childClipFrame = childClip->GetFrame(frame_number); - std::shared_ptr childClipImage = childClipFrame->GetImage(); - - // Scale the original bounding box to this image - QRectF scaledRect = Tracker::scaleAndCenterRect(QRectF(childClipImage->rect()), boxRect); - painter.drawImage(scaledRect, *childClipImage); - } - } - } } } } diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 508107a1d..557098793 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -128,25 +128,6 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f painter.drawRoundedRect(boxRect, bg_corner, bg_corner); } - // Get the image of the Tracked Object' child clip - if (trackedData->ChildClipId() != ""){ - // Cast the parent timeline of this effect - Timeline* parentTimeline = static_cast(ParentTimeline()); - if (parentTimeline){ - // Get the Tracked Object's child clip - Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId()); - if (childClip){ - // Get the image of the child clip for this frame - std::shared_ptr childClipFrame = childClip->GetFrame(frame_number); - std::shared_ptr childClipImage = childClipFrame->GetImage(); - - // Scale the original bounding box to this image - QRectF scaledRect = scaleAndCenterRect(QRectF(childClipImage->rect()), boxRect); - painter.drawImage(scaledRect, *childClipImage); - } - } - } - painter.end(); } From 698e6f474da71c81e8cbe6cf265d32087eade7c5 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 27 Feb 2024 23:21:59 -0600 Subject: [PATCH 10/15] Fix bug with 'Display Box Text' logic being flipped (yes was no, no was yes) --- src/effects/ObjectDetection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 9c0d65b79..4b074a751 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -109,7 +109,7 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i if (trackedObject->draw_box.GetValue(frame_number) == 1) { // Draw bounding box - bool display_text = !display_box_text.GetValue(frame_number); + bool display_text = display_box_text.GetValue(frame_number); std::vector stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number); std::vector bg_rgba = trackedObject->background.GetColorRGBA(frame_number); int stroke_width = trackedObject->stroke_width.GetValue(frame_number); From 3351b52e774122ff412735eedd59add863ce8c01 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 28 Feb 2024 15:29:35 -0600 Subject: [PATCH 11/15] Clear previous parent selection, when switching between Clip and Tracked object --- src/Clip.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Clip.cpp b/src/Clip.cpp index 4da0b8562..3b2599289 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -251,9 +251,11 @@ void Clip::AttachToObject(std::string object_id) // Check for valid tracked object if (trackedObject){ SetAttachedObject(trackedObject); + parentClipObject = NULL; } else if (clipObject) { SetAttachedClip(clipObject); + parentTrackedObject = nullptr; } } } From 620e89440959a8e55c5fccc062d3c12e66256bfc Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 1 Mar 2024 16:47:52 -0600 Subject: [PATCH 12/15] Fixing parentClipObject and parentTrackedObject transform, so the parent clip can be scaled and moved without breaking the tracking. Also refactoring out unneeded complex code left over. --- src/Clip.cpp | 121 +++++++++----------------------- src/TrackedObjectBBox.cpp | 33 --------- src/TrackedObjectBBox.h | 3 - src/TrackedObjectBase.h | 2 - src/effects/ObjectDetection.cpp | 15 ++-- src/effects/Tracker.cpp | 23 ------ src/effects/Tracker.h | 3 - 7 files changed, 41 insertions(+), 159 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 3b2599289..888e17e95 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -789,38 +789,8 @@ std::string Clip::PropertiesJSON(int64_t requested_frame) const { root["waveform"]["choices"].append(add_property_choice_json("Yes", true, waveform)); root["waveform"]["choices"].append(add_property_choice_json("No", false, waveform)); - // Add the parentTrackedObject's properties - if (parentTrackedObject && parentClipObject) - { - // Convert Clip's frame position to Timeline's frame position - long clip_start_position = round(Position() * info.fps.ToDouble()) + 1; - long clip_start_frame = (Start() * info.fps.ToDouble()) + 1; - double timeline_frame_number = requested_frame + clip_start_position - clip_start_frame; - - // Get attached object's parent clip properties - std::map< std::string, float > trackedObjectParentClipProperties = parentTrackedObject->GetParentClipProperties(timeline_frame_number); - double parentObject_frame_number = trackedObjectParentClipProperties["frame_number"]; - // Get attached object properties - std::map< std::string, float > trackedObjectProperties = parentTrackedObject->GetBoxValues(parentObject_frame_number); - - // Correct the parent Tracked Object properties by the clip's reference system - float parentObject_location_x = trackedObjectProperties["cx"] - 0.5 + trackedObjectParentClipProperties["cx"]; - float parentObject_location_y = trackedObjectProperties["cy"] - 0.5 + trackedObjectParentClipProperties["cy"]; - float parentObject_scale_x = trackedObjectProperties["w"]*trackedObjectProperties["sx"]; - float parentObject_scale_y = trackedObjectProperties["h"]*trackedObjectProperties["sy"]; - float parentObject_rotation = trackedObjectProperties["r"] + trackedObjectParentClipProperties["r"]; - - // Add the parent Tracked Object properties to JSON - root["location_x"] = add_property_json("Location X", parentObject_location_x, "float", "", &location_x, -1.0, 1.0, false, requested_frame); - root["location_y"] = add_property_json("Location Y", parentObject_location_y, "float", "", &location_y, -1.0, 1.0, false, requested_frame); - root["scale_x"] = add_property_json("Scale X", parentObject_scale_x, "float", "", &scale_x, 0.0, 1.0, false, requested_frame); - root["scale_y"] = add_property_json("Scale Y", parentObject_scale_y, "float", "", &scale_y, 0.0, 1.0, false, requested_frame); - root["rotation"] = add_property_json("Rotation", parentObject_rotation, "float", "", &rotation, -360, 360, false, requested_frame); - root["shear_x"] = add_property_json("Shear X", shear_x.GetValue(requested_frame), "float", "", &shear_x, -1.0, 1.0, false, requested_frame); - root["shear_y"] = add_property_json("Shear Y", shear_y.GetValue(requested_frame), "float", "", &shear_y, -1.0, 1.0, false, requested_frame); - } // Add the parentClipObject's properties - else if (parentClipObject) + if (parentClipObject) { // Convert Clip's frame position to Timeline's frame position long clip_start_position = round(Position() * info.fps.ToDouble()) + 1; @@ -1345,7 +1315,7 @@ void Clip::apply_waveform(std::shared_ptr frame, QSize timeline_size) { frame->AddImage(source_image); } -// Apply keyframes to the source frame (if any) +// Get QTransform from keyframes QTransform Clip::get_transform(std::shared_ptr frame, int width, int height) { // Get image from clip @@ -1440,62 +1410,41 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig // Get the parentClipObject properties if (parentClipObject){ - - // Convert Clip's frame position to Timeline's frame position - long clip_start_position = round(Position() * info.fps.ToDouble()) + 1; - long clip_start_frame = (Start() * info.fps.ToDouble()) + 1; - double timeline_frame_number = frame->number + clip_start_position - clip_start_frame; + // Get the start trim position of the parent clip + long parent_start_offset = parentClipObject->Start() * info.fps.ToDouble(); + long parent_frame_number = frame->number + parent_start_offset; // Get parent object's properties (Clip) - parentObject_location_x = parentClipObject->location_x.GetValue(timeline_frame_number); - parentObject_location_y = parentClipObject->location_y.GetValue(timeline_frame_number); - parentObject_scale_x = parentClipObject->scale_x.GetValue(timeline_frame_number); - parentObject_scale_y = parentClipObject->scale_y.GetValue(timeline_frame_number); - parentObject_shear_x = parentClipObject->shear_x.GetValue(timeline_frame_number); - parentObject_shear_y = parentClipObject->shear_y.GetValue(timeline_frame_number); - parentObject_rotation = parentClipObject->rotation.GetValue(timeline_frame_number); + parentObject_location_x = parentClipObject->location_x.GetValue(parent_frame_number); + parentObject_location_y = parentClipObject->location_y.GetValue(parent_frame_number); + parentObject_scale_x = parentClipObject->scale_x.GetValue(parent_frame_number); + parentObject_scale_y = parentClipObject->scale_y.GetValue(parent_frame_number); + parentObject_shear_x = parentClipObject->shear_x.GetValue(parent_frame_number); + parentObject_shear_y = parentClipObject->shear_y.GetValue(parent_frame_number); + parentObject_rotation = parentClipObject->rotation.GetValue(parent_frame_number); } - // Get the parentTrackedObject properties - if (parentTrackedObject){ - // Convert Clip's frame position to Timeline's frame position - long clip_start_position = round(Position() * info.fps.ToDouble()) + 1; - long clip_start_frame = (Start() * info.fps.ToDouble()) + 1; - double timeline_frame_number = frame->number + clip_start_position - clip_start_frame; - - // Get parentTrackedObject's parent clip's properties - std::map trackedObjectParentClipProperties = - parentTrackedObject->GetParentClipProperties(timeline_frame_number); - - // Get the attached object's parent clip's properties - if (!trackedObjectParentClipProperties.empty()) - { - // Get parent object's properties (Tracked Object) - float parentObject_frame_number = trackedObjectParentClipProperties["frame_number"]; - - // Access the parentTrackedObject's properties - std::map trackedObjectProperties = parentTrackedObject->GetBoxValues(parentObject_frame_number); - - // Get the Tracked Object's properties and correct them by the clip's reference system - parentObject_location_x = trackedObjectProperties["cx"] - 0.5 + trackedObjectParentClipProperties["location_x"]; - parentObject_location_y = trackedObjectProperties["cy"] - 0.5 + trackedObjectParentClipProperties["location_y"]; - parentObject_scale_x = trackedObjectProperties["w"]*trackedObjectProperties["sx"]; - parentObject_scale_y = trackedObjectProperties["h"]*trackedObjectProperties["sy"]; - parentObject_rotation = trackedObjectProperties["r"] + trackedObjectParentClipProperties["rotation"]; - } - else - { - // Access the parentTrackedObject's properties - std::map trackedObjectProperties = parentTrackedObject->GetBoxValues(timeline_frame_number); - - // Get the Tracked Object's properties and correct them by the clip's reference system - parentObject_location_x = trackedObjectProperties["cx"] - 0.5; - parentObject_location_y = trackedObjectProperties["cy"] - 0.5; - parentObject_scale_x = trackedObjectProperties["w"]*trackedObjectProperties["sx"]; - parentObject_scale_y = trackedObjectProperties["h"]*trackedObjectProperties["sy"]; - parentObject_rotation = trackedObjectProperties["r"]; - } - } + // Get the parentTrackedObject properties + if (parentTrackedObject){ + // Get the attached object's parent clip's properties + Clip* parentClip = (Clip*) parentTrackedObject->ParentClip(); + if (parentClip) + { + // Get the start trim position of the parent clip + long parent_start_offset = parentClip->Start() * info.fps.ToDouble(); + long parent_frame_number = frame->number + parent_start_offset; + + // Access the parentTrackedObject's properties + std::map trackedObjectProperties = parentTrackedObject->GetBoxValues(parent_frame_number); + + // Get the Tracked Object's properties and correct them by the clip's reference system + parentObject_location_x = parentClip->location_x.GetValue(parent_frame_number) + ((trackedObjectProperties["cx"] - 0.5) * parentClip->scale_x.GetValue(parent_frame_number)); + parentObject_location_y = parentClip->location_y.GetValue(parent_frame_number) + ((trackedObjectProperties["cy"] - 0.5) * parentClip->scale_y.GetValue(parent_frame_number)); + parentObject_scale_x = trackedObjectProperties["w"] * trackedObjectProperties["sx"] * parentClip->scale_x.GetValue(parent_frame_number); + parentObject_scale_y = trackedObjectProperties["h"] * trackedObjectProperties["sy"] * parentClip->scale_y.GetValue(parent_frame_number); + parentObject_rotation = trackedObjectProperties["r"] + parentClip->rotation.GetValue(parent_frame_number); + } + } /* GRAVITY LOCATION - Initialize X & Y to the correct values (before applying location curves) */ float x = 0.0; // left @@ -1561,8 +1510,8 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig /* LOCATION, ROTATION, AND SCALE */ float r = rotation.GetValue(frame->number) + parentObject_rotation; // rotate in degrees - x += (width * (location_x.GetValue(frame->number) + parentObject_location_x )); // move in percentage of final width - y += (height * (location_y.GetValue(frame->number) + parentObject_location_y )); // move in percentage of final height + x += width * (location_x.GetValue(frame->number) + parentObject_location_x); // move in percentage of final width + y += height * (location_y.GetValue(frame->number) + parentObject_location_y); // move in percentage of final height float shear_x_value = shear_x.GetValue(frame->number) + parentObject_shear_x; float shear_y_value = shear_y.GetValue(frame->number) + parentObject_shear_y; float origin_x_value = origin_x.GetValue(frame->number); diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index d1a1b5836..6c81a41a3 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -522,36 +522,3 @@ std::map TrackedObjectBBox::GetBoxValues(int64_t frame_numbe return boxValues; } - -// Return a map that contains the properties of this object's parent clip -std::map TrackedObjectBBox::GetParentClipProperties(int64_t frame_number) const { - - // Get the parent clip of this object as a Clip pointer - Clip* parentClip = (Clip *) ParentClip(); - - // Calculate parentClip's frame number - long parentClip_start_position = round( parentClip->Position() * parentClip->info.fps.ToDouble() ) + 1; - long parentClip_start_frame = ( parentClip->Start() * parentClip->info.fps.ToDouble() ) + 1; - float parentClip_frame_number = round(frame_number - parentClip_start_position) + parentClip_start_frame; - - // Get parentClip's Keyframes - float parentClip_location_x = parentClip->location_x.GetValue(parentClip_frame_number); - float parentClip_location_y = parentClip->location_y.GetValue(parentClip_frame_number); - float parentClip_scale_x = parentClip->scale_x.GetValue(parentClip_frame_number); - float parentClip_scale_y = parentClip->scale_y.GetValue(parentClip_frame_number); - float parentClip_rotation = parentClip->rotation.GetValue(parentClip_frame_number); - - // Initialize the parent clip properties map - std::map parentClipProperties; - - // Set the map properties - parentClipProperties["frame_number"] = parentClip_frame_number; - parentClipProperties["timeline_frame_number"] = frame_number; - parentClipProperties["location_x"] = parentClip_location_x; - parentClipProperties["location_y"] = parentClip_location_y; - parentClipProperties["scale_x"] = parentClip_scale_x; - parentClipProperties["scale_y"] = parentClip_scale_y; - parentClipProperties["rotation"] = parentClip_rotation; - - return parentClipProperties; -} diff --git a/src/TrackedObjectBBox.h b/src/TrackedObjectBBox.h index 1569e566c..73ce4b668 100644 --- a/src/TrackedObjectBBox.h +++ b/src/TrackedObjectBBox.h @@ -211,9 +211,6 @@ namespace openshot /// Return a map that contains the bounding box properties and it's keyframes indexed by their names std::map GetBoxValues(int64_t frame_number) const override; - /// Return a map that contains the properties of this object's parent clip - std::map GetParentClipProperties(int64_t frame_number) const override; - }; } // namespace openshot diff --git a/src/TrackedObjectBase.h b/src/TrackedObjectBase.h index faac178aa..4922f1e6f 100644 --- a/src/TrackedObjectBase.h +++ b/src/TrackedObjectBase.h @@ -66,8 +66,6 @@ namespace openshot { virtual void ScalePoints(double scale) { return; }; /// Return the main properties of a TrackedObjectBBox instance - such as position, size and rotation virtual std::map GetBoxValues(int64_t frame_number) const { std::map ret; return ret; }; - /// Return the main properties of the tracked object's parent clip - such as position, size and rotation - virtual std::map GetParentClipProperties(int64_t frame_number) const { std::map ret; return ret; } /// Add a bounding box to the tracked object's BoxVec map virtual void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) { return; }; diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 4b074a751..7bad798e3 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -82,9 +82,6 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform); if (detectionsData.find(frame_number) != detectionsData.end()) { - float fw = frame_image->width(); - float fh = frame_image->height(); - DetectionData detections = detectionsData[frame_number]; for (int i = 0; i < detections.boxes.size(); i++) { if (detections.confidences.at(i) < confidence_threshold || @@ -99,13 +96,13 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i if (trackedObject_it != trackedObjects.end()) { std::shared_ptr trackedObject = std::static_pointer_cast(trackedObject_it->second); - if (trackedObject->Contains(frame_number) && trackedObject->visible.GetValue(frame_number) == 1) { + Clip* parentClip = (Clip*) trackedObject->ParentClip(); + if (parentClip && trackedObject->Contains(frame_number) && trackedObject->visible.GetValue(frame_number) == 1) { BBox trackedBox = trackedObject->GetBox(frame_number); - - QRectF boxRect((trackedBox.cx - trackedBox.width / 2) * fw, - (trackedBox.cy - trackedBox.height / 2) * fh, - trackedBox.width * fw, - trackedBox.height * fh); + QRectF boxRect((trackedBox.cx - trackedBox.width / 2) * frame_image->width(), + (trackedBox.cy - trackedBox.height / 2) * frame_image->height(), + trackedBox.width * frame_image->width(), + trackedBox.height * frame_image->height()); if (trackedObject->draw_box.GetValue(frame_number) == 1) { // Draw bounding box diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 557098793..c4e023e82 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -135,29 +135,6 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f return frame; } -QRectF Tracker::scaleAndCenterRect(const QRectF& sourceRect, const QRectF& targetRect) { - float sourceAspectRatio = sourceRect.width() / sourceRect.height(); - float targetWidth = targetRect.width(); - float targetHeight = targetRect.height(); - float newWidth, newHeight; - - if (sourceAspectRatio > targetRect.width() / targetRect.height()) { - // Source is wider relative to target, so it's constrained by target's width - newWidth = targetWidth; - newHeight = newWidth / sourceAspectRatio; - } else { - // Source is taller relative to target, so it's constrained by target's height - newHeight = targetHeight; - newWidth = newHeight * sourceAspectRatio; - } - - // Center the new rectangle within the target rectangle - float newX = targetRect.left() + (targetWidth - newWidth) / 2.0; - float newY = targetRect.top() + (targetHeight - newHeight) / 2.0; - - return QRectF(newX, newY, newWidth, newHeight); -} - // Get the indexes and IDs of all visible objects in the given frame std::string Tracker::GetVisibleObjects(int64_t frame_number) const{ diff --git a/src/effects/Tracker.h b/src/effects/Tracker.h index c6955a840..d05b72a1f 100644 --- a/src/effects/Tracker.h +++ b/src/effects/Tracker.h @@ -71,9 +71,6 @@ namespace openshot /// Get the indexes and IDs of all visible objects in the given frame std::string GetVisibleObjects(int64_t frame_number) const override; - /// Find a rectangle inside another (centered) - static QRectF scaleAndCenterRect(const QRectF& sourceRect, const QRectF& targetRect); - // Get and Set JSON methods /// Generate JSON string of this object From 8b47373c1ef8ef156a3303a86a6acd16617ca087 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 1 Mar 2024 21:40:55 -0600 Subject: [PATCH 13/15] Fix race condition causing clip to clip parents not to work. Adding a late-bound check, to ensure we have attached the clip on first call. --- src/Clip.cpp | 22 ++++++++++++++++++++-- src/Clip.h | 6 ++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 888e17e95..423f9f17d 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -479,6 +479,24 @@ openshot::EffectBase* Clip::GetEffect(const std::string& id) return nullptr; } +// Return the associated ParentClip (if any) +openshot::Clip* Clip::GetParentClip() { + if (!parentObjectId.empty() && (!parentClipObject && !parentTrackedObject)) { + // Attach parent clip OR object to this clip + AttachToObject(parentObjectId); + } + return parentClipObject; +} + +// Return the associated Parent Tracked Object (if any) +std::shared_ptr Clip::GetParentTrackedObject() { + if (!parentObjectId.empty() && (!parentClipObject && !parentTrackedObject)) { + // Attach parent clip OR object to this clip + AttachToObject(parentObjectId); + } + return parentTrackedObject; +} + // Get file extension std::string Clip::get_file_extension(std::string path) { @@ -1409,7 +1427,7 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig float parentObject_rotation = 0.0; // Get the parentClipObject properties - if (parentClipObject){ + if (GetParentClip()){ // Get the start trim position of the parent clip long parent_start_offset = parentClipObject->Start() * info.fps.ToDouble(); long parent_frame_number = frame->number + parent_start_offset; @@ -1425,7 +1443,7 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig } // Get the parentTrackedObject properties - if (parentTrackedObject){ + if (GetParentTrackedObject()){ // Get the attached object's parent clip's properties Clip* parentClip = (Clip*) parentTrackedObject->ParentClip(); if (parentClip) diff --git a/src/Clip.h b/src/Clip.h index 7bae35ccb..d88ad0817 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -224,6 +224,12 @@ namespace openshot { /// Close the internal reader void Close() override; + /// Return the associated ParentClip (if any) + openshot::Clip* GetParentClip(); + + /// Return the associated Parent Tracked Object (if any) + std::shared_ptr GetParentTrackedObject(); + /// Return the list of effects on the timeline std::list Effects() { return effects; }; From ffb63f55fac1ba61896d9370a24bd3b07efab636 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 3 Mar 2024 16:49:26 -0600 Subject: [PATCH 14/15] Fixing aspect ratio of tracked objects, since the tracked object often is of varying aspect ratios (since the parent clip and tracked object can change over time). Clips which are parented to tracked objects now respect the scale_type (i.e. best fit, stretch, etc...). --- src/Clip.cpp | 122 ++++++++++++++++----------------------------------- src/Clip.h | 5 +-- 2 files changed, 40 insertions(+), 87 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 423f9f17d..3d65b4ad4 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -504,33 +504,6 @@ std::string Clip::get_file_extension(std::string path) return path.substr(path.find_last_of(".") + 1); } -// Reverse an audio buffer -void Clip::reverse_buffer(juce::AudioBuffer* buffer) -{ - int number_of_samples = buffer->getNumSamples(); - int channels = buffer->getNumChannels(); - - // Reverse array (create new buffer to hold the reversed version) - auto *reversed = new juce::AudioBuffer(channels, number_of_samples); - reversed->clear(); - - for (int channel = 0; channel < channels; channel++) - { - int n=0; - for (int s = number_of_samples - 1; s >= 0; s--, n++) - reversed->getWritePointer(channel)[n] = buffer->getWritePointer(channel)[s]; - } - - // Copy the samples back to the original array - buffer->clear(); - // Loop through channels, and get audio samples - for (int channel = 0; channel < channels; channel++) - // Get the audio samples for this channel - buffer->addFrom(channel, 0, reversed->getReadPointer(channel), number_of_samples, 1.0f); - - delete reversed; -} - // Adjust the audio and image of a time mapped frame void Clip::apply_timemapping(std::shared_ptr frame) { @@ -1315,8 +1288,7 @@ void Clip::apply_waveform(std::shared_ptr frame, QSize timeline_size) { std::shared_ptr source_image = frame->GetImage(); // Debug output - ZmqLogger::Instance()->AppendDebugMethod( - "Clip::apply_waveform (Generate Waveform Image)", + ZmqLogger::Instance()->AppendDebugMethod("Clip::apply_waveform (Generate Waveform Image)", "frame->number", frame->number, "Waveform()", Waveform(), "width", timeline_size.width(), @@ -1333,6 +1305,27 @@ void Clip::apply_waveform(std::shared_ptr frame, QSize timeline_size) { frame->AddImage(source_image); } +// Scale a source size to a target size (given a specific scale-type) +QSize Clip::scale_size(QSize source_size, ScaleType source_scale, int target_width, int target_height) { + switch (source_scale) + { + case (SCALE_FIT): { + source_size.scale(target_width, target_height, Qt::KeepAspectRatio); + break; + } + case (SCALE_STRETCH): { + source_size.scale(target_width, target_height, Qt::IgnoreAspectRatio); + break; + } + case (SCALE_CROP): { + source_size.scale(target_width, target_height, Qt::KeepAspectRatioByExpanding);; + break; + } + } + + return source_size; +} + // Get QTransform from keyframes QTransform Clip::get_transform(std::shared_ptr frame, int width, int height) { @@ -1359,63 +1352,13 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig } // Debug output - ZmqLogger::Instance()->AppendDebugMethod( - "Clip::get_transform (Set Alpha & Opacity)", + ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Set Alpha & Opacity)", "alpha_value", alpha_value, "frame->number", frame->number); } /* RESIZE SOURCE IMAGE - based on scale type */ - QSize source_size = source_image->size(); - - switch (scale) - { - case (SCALE_FIT): { - source_size.scale(width, height, Qt::KeepAspectRatio); - - // Debug output - ZmqLogger::Instance()->AppendDebugMethod( - "Clip::get_transform (Scale: SCALE_FIT)", - "frame->number", frame->number, - "source_width", source_size.width(), - "source_height", source_size.height()); - break; - } - case (SCALE_STRETCH): { - source_size.scale(width, height, Qt::IgnoreAspectRatio); - - // Debug output - ZmqLogger::Instance()->AppendDebugMethod( - "Clip::get_transform (Scale: SCALE_STRETCH)", - "frame->number", frame->number, - "source_width", source_size.width(), - "source_height", source_size.height()); - break; - } - case (SCALE_CROP): { - source_size.scale(width, height, Qt::KeepAspectRatioByExpanding); - - // Debug output - ZmqLogger::Instance()->AppendDebugMethod( - "Clip::get_transform (Scale: SCALE_CROP)", - "frame->number", frame->number, - "source_width", source_size.width(), - "source_height", source_size.height()); - break; - } - case (SCALE_NONE): { - // Image is already the original size (i.e. no scaling mode) relative - // to the preview window size (i.e. timeline / preview ratio). No further - // scaling is needed here. - // Debug output - ZmqLogger::Instance()->AppendDebugMethod( - "Clip::get_transform (Scale: SCALE_NONE)", - "frame->number", frame->number, - "source_width", source_size.width(), - "source_height", source_size.height()); - break; - } - } + QSize source_size = scale_size(source_image->size(), scale, width, height); // Initialize parent object's properties (Clip or Tracked Object) float parentObject_location_x = 0.0; @@ -1455,11 +1398,22 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig // Access the parentTrackedObject's properties std::map trackedObjectProperties = parentTrackedObject->GetBoxValues(parent_frame_number); - // Get the Tracked Object's properties and correct them by the clip's reference system + // Get actual scaled parent size + QSize parent_size = scale_size(QSize(parentClip->info.width, parentClip->info.height), + parentClip->scale, width, height); + + // Get actual scaled tracked object size + int trackedWidth = trackedObjectProperties["w"] * trackedObjectProperties["sx"] * parent_size.width() * + parentClip->scale_x.GetValue(parent_frame_number); + int trackedHeight = trackedObjectProperties["h"] * trackedObjectProperties["sy"] * parent_size.height() * + parentClip->scale_y.GetValue(parent_frame_number); + + // Scale the clip source_size based on the actual tracked object size + source_size = scale_size(source_size, scale, trackedWidth, trackedHeight); + + // Update parentObject's properties based on the tracked object's properties and parent clip's scale parentObject_location_x = parentClip->location_x.GetValue(parent_frame_number) + ((trackedObjectProperties["cx"] - 0.5) * parentClip->scale_x.GetValue(parent_frame_number)); parentObject_location_y = parentClip->location_y.GetValue(parent_frame_number) + ((trackedObjectProperties["cy"] - 0.5) * parentClip->scale_y.GetValue(parent_frame_number)); - parentObject_scale_x = trackedObjectProperties["w"] * trackedObjectProperties["sx"] * parentClip->scale_x.GetValue(parent_frame_number); - parentObject_scale_y = trackedObjectProperties["h"] * trackedObjectProperties["sy"] * parentClip->scale_y.GetValue(parent_frame_number); parentObject_rotation = trackedObjectProperties["r"] + parentClip->rotation.GetValue(parent_frame_number); } } diff --git a/src/Clip.h b/src/Clip.h index d88ad0817..caeabd57b 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -160,9 +160,8 @@ namespace openshot { /// Sort effects by order void sort_effects(); - /// Reverse an audio buffer - void reverse_buffer(juce::AudioBuffer* buffer); - + /// Scale a source size to a target size (given a specific scale-type) + QSize scale_size(QSize source_size, ScaleType source_scale, int target_width, int target_height); public: openshot::GravityType gravity; ///< The gravity of a clip determines where it snaps to its parent From 5a0a6a69a822c00899e2ab8d9ccac2e444385a0a Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 5 Mar 2024 10:41:06 -0600 Subject: [PATCH 15/15] Add new "Display All Boxes" Object Detector property, since it can be difficult to turn off each individual box. Made "visible" property read-only on Tracker & Object Detector effects. Improve Keyframe::SetJsonValue method to accept an object or a float. --- src/KeyFrame.cpp | 31 +++++++---- src/TrackedObjectBBox.cpp | 4 +- src/TrackedObjectBase.h | 3 + src/effects/ObjectDetection.cpp | 99 ++++++++++++++++++--------------- src/effects/ObjectDetection.h | 7 ++- 5 files changed, 83 insertions(+), 61 deletions(-) diff --git a/src/KeyFrame.cpp b/src/KeyFrame.cpp index a83648198..1df3f3c06 100644 --- a/src/KeyFrame.cpp +++ b/src/KeyFrame.cpp @@ -374,18 +374,25 @@ void Keyframe::SetJsonValue(const Json::Value root) { Points.clear(); Points.shrink_to_fit(); - if (!root["Points"].isNull()) - // loop through points - for (const auto existing_point : root["Points"]) { - // Create Point - Point p; - - // Load Json into Point - p.SetJsonValue(existing_point); - - // Add Point to Keyframe - AddPoint(p); - } + if (root.isObject() && !root["Points"].isNull()) { + // loop through points in JSON Object + for (const auto existing_point : root["Points"]) { + // Create Point + Point p; + + // Load Json into Point + p.SetJsonValue(existing_point); + + // Add Point to Keyframe + AddPoint(p); + } + } else if (root.isNumeric()) { + // Create Point from Numeric value + Point p(root.asFloat()); + + // Add Point to Keyframe + AddPoint(p); + } } // Get the change in Y value (from the previous Y value) diff --git a/src/TrackedObjectBBox.cpp b/src/TrackedObjectBBox.cpp index 6c81a41a3..b520b0549 100644 --- a/src/TrackedObjectBBox.cpp +++ b/src/TrackedObjectBBox.cpp @@ -431,9 +431,7 @@ Json::Value TrackedObjectBBox::PropertiesJSON(int64_t requested_frame) const root["scale_x"] = add_property_json("Scale (Width)", scale_x.GetValue(requested_frame), "float", "", &scale_x, 0.0, 1.0, false, requested_frame); root["scale_y"] = add_property_json("Scale (Height)", scale_y.GetValue(requested_frame), "float", "", &scale_y, 0.0, 1.0, false, requested_frame); root["rotation"] = add_property_json("Rotation", rotation.GetValue(requested_frame), "float", "", &rotation, 0, 360, false, requested_frame); - root["visible"] = add_property_json("Visible", visible.GetValue(requested_frame), "int", "", &visible, 0, 1, false, requested_frame); - root["visible"]["choices"].append(add_property_choice_json("Yes", true, visible.GetValue(requested_frame))); - root["visible"]["choices"].append(add_property_choice_json("No", false, visible.GetValue(requested_frame))); + root["visible"] = add_property_json("Visible", visible.GetValue(requested_frame), "int", "", &visible, 0, 1, true, requested_frame); root["draw_box"] = add_property_json("Draw Box", draw_box.GetValue(requested_frame), "int", "", &draw_box, 0, 1, false, requested_frame); root["draw_box"]["choices"].append(add_property_choice_json("Yes", true, draw_box.GetValue(requested_frame))); diff --git a/src/TrackedObjectBase.h b/src/TrackedObjectBase.h index 4922f1e6f..58b312ed5 100644 --- a/src/TrackedObjectBase.h +++ b/src/TrackedObjectBase.h @@ -39,7 +39,10 @@ namespace openshot { public: + /// Keyframe to track if a box is visible in the current frame (read-only) Keyframe visible; + + /// Keyframe to determine if a specific box is drawn (or hidden) Keyframe draw_box; /// Default constructor diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 7bad798e3..5d45992be 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -30,7 +30,8 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -ObjectDetection::ObjectDetection(std::string clipObDetectDataPath) +ObjectDetection::ObjectDetection(std::string clipObDetectDataPath) : +display_box_text(1.0), display_boxes(1.0) { // Init effect properties init_effect_details(); @@ -43,7 +44,8 @@ ObjectDetection::ObjectDetection(std::string clipObDetectDataPath) } // Default constructor -ObjectDetection::ObjectDetection() +ObjectDetection::ObjectDetection() : + display_box_text(1.0), display_boxes(1.0) { // Init effect properties init_effect_details(); @@ -104,55 +106,54 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i trackedBox.width * frame_image->width(), trackedBox.height * frame_image->height()); - if (trackedObject->draw_box.GetValue(frame_number) == 1) { - // Draw bounding box - bool display_text = display_box_text.GetValue(frame_number); - std::vector stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number); - std::vector bg_rgba = trackedObject->background.GetColorRGBA(frame_number); - int stroke_width = trackedObject->stroke_width.GetValue(frame_number); - float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number); - float bg_alpha = trackedObject->background_alpha.GetValue(frame_number); - float bg_corner = trackedObject->background_corner.GetValue(frame_number); - - // Set the pen for the border - QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha)); - pen.setWidth(stroke_width); - painter.setPen(pen); - - // Set the brush for the background - QBrush brush(QColor(bg_rgba[0], bg_rgba[1], bg_rgba[2], 255 * bg_alpha)); - painter.setBrush(brush); - - // Draw the rounded rectangle + // Get properties of tracked object (i.e. colors, stroke width, etc...) + std::vector stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number); + std::vector bg_rgba = trackedObject->background.GetColorRGBA(frame_number); + int stroke_width = trackedObject->stroke_width.GetValue(frame_number); + float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number); + float bg_alpha = trackedObject->background_alpha.GetValue(frame_number); + float bg_corner = trackedObject->background_corner.GetValue(frame_number); + + // Set the pen for the border + QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha)); + pen.setWidth(stroke_width); + painter.setPen(pen); + + // Set the brush for the background + QBrush brush(QColor(bg_rgba[0], bg_rgba[1], bg_rgba[2], 255 * bg_alpha)); + painter.setBrush(brush); + + if (display_boxes.GetValue(frame_number) == 1 && trackedObject->draw_box.GetValue(frame_number) == 1) { + // Only draw boxes if both properties are set to YES (draw all boxes, and draw box of the selected box) painter.drawRoundedRect(boxRect, bg_corner, bg_corner); + } - if(display_text) { - // Draw text label above bounding box - // Get the confidence and classId for the current detection - int classId = detections.classIds.at(i); + if(display_box_text.GetValue(frame_number) == 1) { + // Draw text label above bounding box + // Get the confidence and classId for the current detection + int classId = detections.classIds.at(i); - // Get the label for the class name and its confidence - QString label = QString::number(objectId); - if (!classNames.empty()) { - label = QString::fromStdString(classNames[classId]) + ":" + label; - } + // Get the label for the class name and its confidence + QString label = QString::number(objectId); + if (!classNames.empty()) { + label = QString::fromStdString(classNames[classId]) + ":" + label; + } - // Set up the painter, font, and pen - QFont font; - font.setPixelSize(14); - painter.setFont(font); + // Set up the painter, font, and pen + QFont font; + font.setPixelSize(14); + painter.setFont(font); - // Calculate the size of the text - QFontMetrics fontMetrics(font); - QSize labelSize = fontMetrics.size(Qt::TextSingleLine, label); + // Calculate the size of the text + QFontMetrics fontMetrics(font); + QSize labelSize = fontMetrics.size(Qt::TextSingleLine, label); - // Define the top left point of the rectangle - double left = boxRect.center().x() - (labelSize.width() / 2.0); - double top = std::max(static_cast(boxRect.top()), labelSize.height()) - 4.0; + // Define the top left point of the rectangle + double left = boxRect.center().x() - (labelSize.width() / 2.0); + double top = std::max(static_cast(boxRect.top()), labelSize.height()) - 4.0; - // Draw the text - painter.drawText(QPointF(left, top), label); - } + // Draw the text + painter.drawText(QPointF(left, top), label); } } } @@ -343,6 +344,7 @@ Json::Value ObjectDetection::JsonValue() const { root["selected_object_index"] = selectedObjectIndex; root["confidence_threshold"] = confidence_threshold; root["display_box_text"] = display_box_text.JsonValue(); + root["display_boxes"] = display_boxes.JsonValue(); // Add tracked object's IDs to root Json::Value objects; @@ -399,6 +401,9 @@ void ObjectDetection::SetJsonValue(const Json::Value root) { if (!root["display_box_text"].isNull()) display_box_text.SetJsonValue(root["display_box_text"]); + if (!root["display_boxes"].isNull()) + display_boxes.SetJsonValue(root["display_boxes"]); + if (!root["class_filter"].isNull()) { class_filter = root["class_filter"].asString(); @@ -458,10 +463,14 @@ std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const { root["confidence_threshold"] = add_property_json("Confidence Theshold", confidence_threshold, "float", "", NULL, 0, 1, false, requested_frame); root["class_filter"] = add_property_json("Class Filter", 0.0, "string", class_filter, NULL, -1, -1, false, requested_frame); - root["display_box_text"] = add_property_json("Draw Box Text", display_box_text.GetValue(requested_frame), "int", "", &display_box_text, 0, 1, false, requested_frame); + root["display_box_text"] = add_property_json("Draw All Text", display_box_text.GetValue(requested_frame), "int", "", &display_box_text, 0, 1, false, requested_frame); root["display_box_text"]["choices"].append(add_property_choice_json("Yes", true, display_box_text.GetValue(requested_frame))); root["display_box_text"]["choices"].append(add_property_choice_json("No", false, display_box_text.GetValue(requested_frame))); + root["display_boxes"] = add_property_json("Draw All Boxes", display_boxes.GetValue(requested_frame), "int", "", &display_boxes, 0, 1, false, requested_frame); + root["display_boxes"]["choices"].append(add_property_choice_json("Yes", true, display_boxes.GetValue(requested_frame))); + root["display_boxes"]["choices"].append(add_property_choice_json("No", false, display_boxes.GetValue(requested_frame))); + // Return formatted string return root.toStyledString(); } diff --git a/src/effects/ObjectDetection.h b/src/effects/ObjectDetection.h index 42f278d10..d56eca721 100644 --- a/src/effects/ObjectDetection.h +++ b/src/effects/ObjectDetection.h @@ -62,10 +62,15 @@ namespace openshot std::vector classNames; std::vector classesColor; - /// Draw class name and confidence score on top of the bounding box + /// Draw ALL class name and ID #'s on top of the bounding boxes (or hide all text) Keyframe display_box_text; + + /// Draw ALL tracked bounding boxes (or hide all boxes) + Keyframe display_boxes; + /// Minimum confidence value to display the detected objects float confidence_threshold = 0.5; + /// Contain the user selected classes for visualization std::vector display_classes; std::string class_filter;