diff --git a/include/CVObjectDetection.h b/include/CVObjectDetection.h index db07a5cc8..36086fd83 100644 --- a/include/CVObjectDetection.h +++ b/include/CVObjectDetection.h @@ -44,11 +44,13 @@ #include "Clip.h" #include "objdetectdata.pb.h" +#include "../src/sort_filter/sort.hpp" + using google::protobuf::util::TimeUtil; struct CVDetectionData{ CVDetectionData(){} - CVDetectionData(std::vector _classIds, std::vector _confidences, std::vector _boxes, size_t _frameId){ + CVDetectionData(std::vector _classIds, std::vector _confidences, std::vector> _boxes, size_t _frameId){ classIds = _classIds; confidences = _confidences; boxes = _boxes; @@ -57,7 +59,7 @@ struct CVDetectionData{ size_t frameId; std::vector classIds; std::vector confidences; - std::vector boxes; + std::vector> boxes; }; class CVObjectDetection{ @@ -74,6 +76,8 @@ class CVObjectDetection{ std::string processingDevice; std::string protobuf_data_path; + SortTracker sort; + uint progress; size_t start; @@ -86,6 +90,8 @@ class CVObjectDetection{ void DetectObjects(const cv::Mat &frame, size_t frame_number); + bool iou(cv::Rect pred_box, cv::Rect sort_box); + // Remove the bounding boxes with low confidence using non-maxima suppression void postprocess(const cv::Size &frameDims, const std::vector& out, size_t frame_number); @@ -100,6 +106,8 @@ class CVObjectDetection{ void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false); + CVDetectionData GetDetectionData(size_t frameId); + /// Protobuf Save and Load methods // Save protobuf file bool SaveTrackedData(); diff --git a/include/CVStabilization.h b/include/CVStabilization.h index 91638ed47..45e15c605 100644 --- a/include/CVStabilization.h +++ b/include/CVStabilization.h @@ -86,8 +86,7 @@ class CVStabilization { size_t end; cv::Mat last_T; - cv::Mat cur, cur_grey; - cv::Mat prev, prev_grey; + cv::Mat prev_grey; std::vector prev_to_cur_transform; // Previous to current std::string protobuf_data_path; diff --git a/include/CVTracker.h b/include/CVTracker.h index 9e52d5d2b..d4f42a970 100644 --- a/include/CVTracker.h +++ b/include/CVTracker.h @@ -28,10 +28,10 @@ using google::protobuf::util::TimeUtil; struct FrameData{ size_t frame_id = -1; float rotation = 0; - int x1 = -1; - int y1 = -1; - int x2 = -1; - int y2 = -1; + float x1 = -1; + float y1 = -1; + float x2 = -1; + float y2 = -1; // Constructors FrameData() @@ -40,7 +40,7 @@ struct FrameData{ FrameData( size_t _frame_id) {frame_id = _frame_id;} - FrameData( size_t _frame_id , float _rotation, int _x1, int _y1, int _x2, int _y2) + FrameData( size_t _frame_id , float _rotation, float _x1, float _y1, float _x2, float _y2) { frame_id = _frame_id; rotation = _rotation; @@ -51,17 +51,6 @@ struct FrameData{ } }; -class RemoveJitter{ - private: - std::vector bboxTracker; - int boxesInterval; - int boxesInVector; - - public: - RemoveJitter(int boxesInterval); - void update(cv::Rect2d bbox, cv::Rect2d &out_bbox); -}; - class CVTracker { private: std::map trackedDataById; // Save tracked data @@ -69,6 +58,7 @@ class CVTracker { cv::Ptr tracker; // Pointer of the selected tracker cv::Rect2d bbox; // Bounding box coords + SortTracker sort; std::string protobuf_data_path; // Path to protobuf data file @@ -86,7 +76,7 @@ class CVTracker { bool initTracker(cv::Mat &frame, size_t frameId); // Update the object tracker according to frame - bool trackFrame(cv::Mat &frame, size_t frameId, SortTracker &sort, RemoveJitter &removeJitter); + bool trackFrame(cv::Mat &frame, size_t frameId); public: diff --git a/include/effects/ObjectDetection.h b/include/effects/ObjectDetection.h index 772ef6827..81afa6854 100644 --- a/include/effects/ObjectDetection.h +++ b/include/effects/ObjectDetection.h @@ -43,7 +43,7 @@ struct DetectionData{ DetectionData(){} - DetectionData(std::vector _classIds, std::vector _confidences, std::vector _boxes, size_t _frameId){ + DetectionData(std::vector _classIds, std::vector _confidences, std::vector> _boxes, size_t _frameId){ classIds = _classIds; confidences = _confidences; boxes = _boxes; @@ -52,7 +52,7 @@ struct DetectionData{ size_t frameId; std::vector classIds; std::vector confidences; - std::vector boxes; + std::vector> boxes; }; namespace openshot diff --git a/include/effects/Tracker.h b/include/effects/Tracker.h index c237a11d6..5421096ea 100644 --- a/include/effects/Tracker.h +++ b/include/effects/Tracker.h @@ -52,10 +52,10 @@ using google::protobuf::util::TimeUtil; struct EffectFrameData{ size_t frame_id = -1; float rotation = 0; - int x1 = -1; - int y1 = -1; - int x2 = -1; - int y2 = -1; + float x1 = -1; + float y1 = -1; + float x2 = -1; + float y2 = -1; // Constructors EffectFrameData() @@ -64,7 +64,7 @@ struct EffectFrameData{ EffectFrameData( int _frame_id) {frame_id = _frame_id;} - EffectFrameData( int _frame_id , float _rotation, int _x1, int _y1, int _x2, int _y2) + EffectFrameData( int _frame_id , float _rotation, float _x1, float _y1, float _x2, float _y2) { frame_id = _frame_id; rotation = _rotation; diff --git a/src/CVObjectDetection.cpp b/src/CVObjectDetection.cpp index d124c3acd..651d880ef 100644 --- a/src/CVObjectDetection.cpp +++ b/src/CVObjectDetection.cpp @@ -40,7 +40,6 @@ CVObjectDetection::CVObjectDetection(std::string processInfoJson, ProcessingController &processingController) : processingController(&processingController), processingDevice("CPU"){ SetJson(processInfoJson); - setProcessingDevice(); } void CVObjectDetection::setProcessingDevice(){ @@ -70,7 +69,10 @@ void CVObjectDetection::detectObjectsClip(openshot::Clip &video, size_t _start, nmsThreshold = 0.1; // Load the network + if(classesFile == "" || modelConfiguration == "" || modelWeights == "") + return; net = cv::dnn::readNetFromDarknet(modelConfiguration, modelWeights); + setProcessingDevice(); size_t frame_number; if(!process_interval || end == 0 || end-start <= 0){ @@ -164,7 +166,100 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector std::vector indices; cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); - detectionsData[frameId] = CVDetectionData(classIds, confidences, boxes, frameId); + // std::vector sortBoxes; + // for(auto box : boxes) + // sortBoxes.push_back(box); + // sort.update(sortBoxes, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2))); + + + // sortBoxes.clear(); + // for(auto TBox : sort.frameTrackingResult) + // if(TBox.frame == frameId){ + // sortBoxes.push_back(TBox.box); + // } + + // for(int i = 0; i > rectAndClasses; + // for(int i=0; i bboxes; + // rectAndClasses[classIds[i]] = bboxes; + // } + + // rectAndClasses[classIds[i]].push_back(boxes[i]); + // } + + // for(std::map >::iterator it = rectAndClasses.begin(); it != rectAndClasses.end(); it++){ + // if(sort.find(it->first) == sort.end()){ + // SortTracker classTracker; + // sort[it->first] = classTracker; + // } + // sort[it->first].update(it->second, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2))); + // } + + // classIds.clear(); boxes.clear(); confidences.clear(); + + // for(std::map::iterator it = sort.begin(); it != sort.end(); it++){ + // for(auto TBox : it->second.frameTrackingResult){ + // boxes.push_back(TBox.box); + // classIds.push_back(it->first); + // confidences.push_back(1); + // } + // } + + + std::vector> normalized_boxes; + for(auto box : boxes){ + cv::Rect_ normalized_box; + normalized_box.x = (box.x)/(float)frameDims.width; + normalized_box.y = (box.y)/(float)frameDims.height; + normalized_box.width = (box.x+box.width)/(float)frameDims.width; + normalized_box.height = (box.y+box.height)/(float)frameDims.height; + normalized_boxes.push_back(normalized_box); + } + + detectionsData[frameId] = CVDetectionData(classIds, confidences, normalized_boxes, frameId); +} + +bool CVObjectDetection::iou(cv::Rect pred_box, cv::Rect sort_box){ + // determine the (x, y)-coordinates of the intersection rectangle + int xA = std::max(pred_box.x, sort_box.x); + int yA = std::max(pred_box.y, sort_box.y); + int xB = std::min(pred_box.x + pred_box.width, sort_box.x + sort_box.width); + int yB = std::min(pred_box.y + pred_box.height, sort_box.y + sort_box.height); + + // compute the area of intersection rectangle + int interArea = std::max(0, xB - xA + 1) * std::max(0, yB - yA + 1); + // compute the area of both the prediction and ground-truth + // rectangles + int boxAArea = (pred_box.width + 1) * (pred_box.height + 1); + int boxBArea = (sort_box.width + 1) * (sort_box.height + 1); + // compute the intersection over union by taking the intersection + // area and dividing it by the sum of prediction + ground-truth + // areas - the interesection area + float iou = interArea / (float)(boxAArea + boxBArea - interArea); + + if(iou > 0.75) + return true; + return false; } // Get the names of the output layers @@ -185,6 +280,17 @@ std::vector CVObjectDetection::getOutputsNames(const cv::dnn::Net& n return names; } +CVDetectionData CVObjectDetection::GetDetectionData(size_t frameId){ + // Check if the stabilizer info for the requested frame exists + if ( detectionsData.find(frameId) == detectionsData.end() ) { + + return CVDetectionData(); + } else { + + return detectionsData[frameId]; + } +} + bool CVObjectDetection::SaveTrackedData(){ // Create tracker message libopenshotobjdetect::ObjDetect objMessage; diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp index 0283b1c30..944f222a9 100644 --- a/src/CVStabilization.cpp +++ b/src/CVStabilization.cpp @@ -82,6 +82,12 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t // Track current frame features and find the relative transformation void CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ + std::cout<<"frame "< prev_corner2, cur_corner2; std::vector status; std::vector err; - // Extract new image features cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30); // Track features @@ -105,23 +110,37 @@ void CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ cur_corner2.push_back(cur_corner[i]); } } + // In case no feature was detected + if(prev_corner2.empty() || cur_corner2.empty()){ + last_T = cv::Mat(); + prev_grey = cv::Mat(); + return; + } + // Translation + rotation only - cv::Mat T = estimateRigidTransform(prev_corner2, cur_corner2, false); // false = rigid transform, no scaling/shearing + cv::Mat T = estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing - // If no transformation is found, just use the last known good transform. - if(T.data == NULL) { - last_T.copyTo(T); + double da, dx, dy; + if(T.size().width == 0 || T.size().height == 0){ + + dx = 0; + dy = 0; + da = 0; + } + else{ + // If no transformation is found, just use the last known good transform. + if(T.data == NULL && !last_T.empty()) + last_T.copyTo(T); + // Decompose T + dx = T.at(0,2); + dy = T.at(1,2); + da = atan2(T.at(1,0), T.at(0,0)); } T.copyTo(last_T); - // Decompose T - double dx = T.at(0,2); - double dy = T.at(1,2); - double da = atan2(T.at(1,0), T.at(0,0)); prev_to_cur_transform.push_back(TransformParam(dx, dy, da)); - - cur.copyTo(prev); + std::cout<<"10\n"; frame.copyTo(prev_grey); // Show processing info diff --git a/src/CVTracker.cpp b/src/CVTracker.cpp index 203dc8ad4..9d1eb9a07 100644 --- a/src/CVTracker.cpp +++ b/src/CVTracker.cpp @@ -81,9 +81,6 @@ void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, boo bool trackerInit = false; - SortTracker sort; - RemoveJitter removeJitter(0); - size_t frame; // Loop through video for (frame = start; frame <= end; frame++) @@ -112,7 +109,7 @@ void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, boo } else{ // Update the object tracker according to frame - trackerInit = trackFrame(cvimage, frame_number, sort, removeJitter); + trackerInit = trackFrame(cvimage, frame_number); // Draw box on image FrameData fd = GetTrackedData(frame_number); @@ -132,14 +129,20 @@ bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){ // Initialize tracker tracker->init(frame, bbox); + float fw = frame.size().width; + float fh = frame.size().height; + // Add new frame data - trackedDataById[frameId] = FrameData(frameId, 0, bbox.x, bbox.y, bbox.x+bbox.width, bbox.y+bbox.height); + trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw, + (bbox.y)/fh, + (bbox.x+bbox.width)/fw, + (bbox.y+bbox.height)/fh); return true; } // Update the object tracker according to frame -bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId, SortTracker &sort, RemoveJitter &removeJitter){ +bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){ // Update the tracking result bool ok = tracker->update(frame, bbox); @@ -147,17 +150,21 @@ bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId, SortTracker &sort, Re // Otherwise add only frame number if (ok) { + float fw = frame.size().width; + float fh = frame.size().height; + std::vector bboxes = {bbox}; sort.update(bboxes, frameId, sqrt(pow(frame.rows, 2) + pow(frame.cols, 2))); for(auto TBox : sort.frameTrackingResult) - bbox = TBox.box; - - // removeJitter.update(bbox, bbox); + bbox = TBox.box; // Add new frame data - trackedDataById[frameId] = FrameData(frameId, 0, bbox.x, bbox.y, bbox.x+bbox.width, bbox.y+bbox.height); + trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw, + (bbox.y)/fh, + (bbox.x+bbox.width)/fw, + (bbox.y+bbox.height)/fh); } else { @@ -240,10 +247,10 @@ bool CVTracker::LoadTrackedData(){ // Load bounding box data const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box(); - int x1 = box.x1(); - int y1 = box.y1(); - int x2 = box.x2(); - int y2 = box.y2(); + float x1 = box.x1(); + float y1 = box.y1(); + float x2 = box.x2(); + float y2 = box.y2(); // Assign data to tracker map trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2); @@ -315,42 +322,3 @@ void CVTracker::SetJsonValue(const Json::Value root) { } } -RemoveJitter::RemoveJitter(int boxesInterval) : boxesInterval(boxesInterval), boxesInVector(0){ -} - -void RemoveJitter::update(cv::Rect2d bbox, cv::Rect2d &out_bbox){ - - bboxTracker.push_back(bbox); - - // Just to initialize the vector properly - if(boxesInVector < boxesInterval+1){ - boxesInVector++; - out_bbox = bbox; - } - else{ - cv::Rect2d old_bbox = bboxTracker.front(); - cv::Rect2d new_bbox = bboxTracker.back(); - - int centerX_1 = old_bbox.x + old_bbox.width/2; - int centerY_1 = old_bbox.y + old_bbox.height/2; - int centerX_2 = new_bbox.x + new_bbox.width/2; - int centerY_2 = new_bbox.y + new_bbox.height/2; - - int dif_centerXs = abs(centerX_1 - centerX_2); - int dif_centerYs = abs(centerY_1 - centerY_2); - - cout< 6 || dif_centerYs > 6){ - out_bbox = new_bbox; - } - else{ - cv::Rect2d mean_bbox((old_bbox.x + new_bbox.x)/2, (old_bbox.y + new_bbox.y)/2, (old_bbox.width + new_bbox.width)/2, (old_bbox.height + new_bbox.height)/2); - out_bbox = mean_bbox; - } - - bboxTracker.erase(bboxTracker.begin()); - } -} - diff --git a/src/ClipProcessingJobs.cpp b/src/ClipProcessingJobs.cpp index 41eaa2b1f..80c44c0ec 100644 --- a/src/ClipProcessingJobs.cpp +++ b/src/ClipProcessingJobs.cpp @@ -42,11 +42,11 @@ void ClipProcessingJobs::trackClip(Clip& clip, ProcessingController& controller) } -// Apply stabilization to clip +// Apply object detection to clip void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& controller){ - // create CVStabilization object + // create CVObjectDetection object CVObjectDetection objDetector(processInfoJson, controller); - // Start stabilization process + // Start object detection process objDetector.detectObjectsClip(clip); // Thread controller. If effect processing is done, save data @@ -56,7 +56,7 @@ void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& con return; } else{ - // Save stabilization data + // Save object detection data objDetector.SaveTrackedData(); // tells to UI that the processing finished controller.SetFinished(true); diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index ba332cf7e..20cfa12d7 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -29,6 +29,37 @@ */ #include "../../include/effects/ObjectDetection.h" + +/** + * @file + * @brief Source file for Tracker effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + #include "../../include/effects/Tracker.h" using namespace openshot; @@ -77,11 +108,18 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i // Check if track data exists for the requested frame if (detectionsData.find(frame_number) != detectionsData.end()) { + float fw = cv_image.size().width; + float fh = cv_image.size().height; DetectionData detections = detectionsData[frame_number]; for(int i = 0; i bb_nrml = detections.boxes.at(i); + cv::Rect2d box((int)(bb_nrml.x*fw), + (int)(bb_nrml.y*fh), + (int)((bb_nrml.width - bb_nrml.x)*fw), + (int)((bb_nrml.height - bb_nrml.y)*fh)); drawPred(detections.classIds.at(i), detections.confidences.at(i), - detections.boxes.at(i), cv_image); + box, cv_image); } } } @@ -148,17 +186,17 @@ bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){ std::vector classIds; std::vector confidences; - std::vector boxes; + std::vector> boxes; for(int i = 0; i < pbFrameData.bounding_box_size(); i++){ - int x1 = box.at(i).x1(); - int y1 = box.at(i).y1(); - int x2 = box.at(i).x2(); - int y2 = box.at(i).y2(); - int classId = box.at(i).classid(); - float confidence = box.at(i).confidence(); - - cv::Rect2d box(x1, y1, x2-x1, y2-y1); + float x1 = box.Get(i).x1(); + float y1 = box.Get(i).y1(); + float x2 = box.Get(i).x2(); + float y2 = box.Get(i).y2(); + int classId = box.Get(i).classid(); + float confidence = box.Get(i).confidence(); + + cv::Rect_ box(x1, y1, x2-x1, y2-y1); boxes.push_back(box); classIds.push_back(classId); diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index b2f479d84..43bb9e5c5 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -68,6 +68,7 @@ void Tracker::init_effect_details() // modified openshot::Frame object std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t frame_number) { + // Get the frame's image cv::Mat frame_image = frame->GetImageCV(); @@ -77,9 +78,15 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f // Check if track data exists for the requested frame if (trackedDataById.find(frame_number) != trackedDataById.end()) { + float fw = frame_image.size().width; + float fh = frame_image.size().height; + // Draw box on image EffectFrameData fd = trackedDataById[frame_number]; - cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); + cv::Rect2d box((int)(fd.x1*fw), + (int)(fd.y1*fh), + (int)((fd.x2-fd.x1)*fw), + (int)((fd.y2-fd.y1)*fh)); cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); } } @@ -118,10 +125,10 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ // Load bounding box data const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box(); - int x1 = box.x1(); - int y1 = box.y1(); - int x2 = box.x2(); - int y2 = box.y2(); + float x1 = box.x1(); + float y1 = box.y1(); + float x2 = box.x2(); + float y2 = box.y2(); // Assign data to tracker map trackedDataById[id] = EffectFrameData(id, rotation, x1, y1, x2, y2); diff --git a/src/examples/Example_opencv.cpp b/src/examples/Example_opencv.cpp index 2d196710d..7a2caf2a8 100644 --- a/src/examples/Example_opencv.cpp +++ b/src/examples/Example_opencv.cpp @@ -94,7 +94,7 @@ int main(int argc, char* argv[]) { // Get media path std::stringstream path; - path << TEST_MEDIA_PATH << ((OBJECT_DETECTION_DATA) ? "test_video.mp4" : "test.avi"); + path << TEST_MEDIA_PATH << ((OBJECT_DETECTION_DATA) ? "run.mp4" : "test.avi"); // test_video.mp4 --> Used for object detector // test.avi --> Used for tracker and stabilizer diff --git a/src/examples/run.mp4 b/src/examples/run.mp4 new file mode 100644 index 000000000..d54995807 Binary files /dev/null and b/src/examples/run.mp4 differ diff --git a/src/protobuf_messages/objdetectdata.proto b/src/protobuf_messages/objdetectdata.proto index 6ff87c8c8..7cd4d79f1 100644 --- a/src/protobuf_messages/objdetectdata.proto +++ b/src/protobuf_messages/objdetectdata.proto @@ -11,10 +11,10 @@ message Frame { int32 id = 1; // Frame ID. message Box{ - int32 x1 = 1; - int32 y1 = 2; - int32 x2 = 3; - int32 y2 = 4; + float x1 = 1; + float y1 = 2; + float x2 = 3; + float y2 = 4; int32 classId = 5; float confidence = 6; } diff --git a/src/protobuf_messages/trackerdata.proto b/src/protobuf_messages/trackerdata.proto index 3511d67a9..7b3b1a960 100644 --- a/src/protobuf_messages/trackerdata.proto +++ b/src/protobuf_messages/trackerdata.proto @@ -12,10 +12,10 @@ message Frame { float rotation = 2; message Box { - int32 x1 = 1; - int32 y1 = 2; - int32 x2 = 3; - int32 y2 = 4; + float x1 = 1; + float y1 = 2; + float x2 = 3; + float y2 = 4; } Box bounding_box = 3; diff --git a/src/sort_filter/sort.cpp b/src/sort_filter/sort.cpp index c89151b15..44daf6761 100644 --- a/src/sort_filter/sort.cpp +++ b/src/sort_filter/sort.cpp @@ -100,7 +100,6 @@ void SortTracker::update(vector detections_cv, int frame_count, double for (unsigned int j = 0; j < detNum; j++) { // use 1-iou because the hungarian algorithm computes a minimum-cost assignment. - // TODO: change norm hardcode to proportion of image diagonal double distance = SortTracker::GetCentroidsDistance(predictedBoxes[i], detections[j].box) / image_diagonal; centroid_dist_matrix[i][j] = distance; } diff --git a/src/sort_filter/sort.hpp b/src/sort_filter/sort.hpp index c21b0e52d..eaa313375 100644 --- a/src/sort_filter/sort.hpp +++ b/src/sort_filter/sort.hpp @@ -13,10 +13,13 @@ #ifndef _OPENCV_KCFTRACKER_HPP_ #define _OPENCV_KCFTRACKER_HPP_ #endif +#pragma once typedef struct TrackingBox { int frame = 0; + float confidence = 0; + int classId = 0; int id = 0; cv::Rect_ box = cv::Rect_(0.0, 0.0, 0.0, 0.0); } TrackingBox; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 846895f1d..08c8dc8e8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -141,7 +141,9 @@ set(OPENSHOT_TEST_FILES ########## SET OPENCV RELATED TEST FILES ############### set(OPENSHOT_CV_TEST_FILES CVTracker_Tests.cpp - CVStabilizer_Tests.cpp) + CVStabilizer_Tests.cpp + CVObjectDetection_Tests.cpp + ) ################ TESTER EXECUTABLE ################# diff --git a/tests/CVObjectDetection_Tests.cpp b/tests/CVObjectDetection_Tests.cpp new file mode 100644 index 000000000..baf7cabc3 --- /dev/null +++ b/tests/CVObjectDetection_Tests.cpp @@ -0,0 +1,114 @@ +/** + * @file + * @brief Unit tests for openshot::Frame + * @author Jonathan Thomas + * @author FeRD (Frank Dana) + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "UnitTest++.h" +// Prevent name clashes with juce::UnitTest +#define DONT_SET_USING_JUCE_NAMESPACE 1 +#include "../include/OpenShot.h" +#include "../include/ProcessingController.h" +#include + +using namespace openshot; + +SUITE(CVObjectDetection_Tests) +{ + + // Just for the stabilizer constructor, it won't be used + ProcessingController processingController; + + TEST(DetectObject_Video) + { + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test_video.mp4"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + CVObjectDetection objectDetector("\"processing_device\": \"GPU\"", processingController); + + objectDetector.detectObjectsClip(c1, 0, 100, true); + + CVDetectionData dd = objectDetector.GetDetectionData(20); + + // int x1 = dd.boxes[20].x; + // int y1 = dd.boxes[20].y; + // int x2 = x1 + dd.boxes[20].width(); + // int y2 = y2 + dd.boxes[20].height(); + // float confidence = dd.confidences[20]; + // int classId = dd.classIds[20]; + + } + + + TEST(SaveLoad_Protobuf) + { + + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test_video.mp4"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + CVObjectDetection objectDetector_1("{\"protobuf_data_path\": \"object_detector.data\", \"processing_device\": \"GPU\"}", processingController); + + objectDetector_1.detectObjectsClip(c1, 0, 100, true); + + CVDetectionData dd_1 = objectDetector_1.GetDetectionData(20); + + objectDetector_1.SaveTrackedData(); + + CVObjectDetection objectDetector_2("{\"protobuf_data_path\": \"object_detector.data\", \"processing_device\": \"\"}", processingController); + + // objectDetector_2.LoadTrackedData(); + + CVDetectionData dd_2 = objectDetector_2.GetDetectionData(20); + + // int x1_1 = dd_1.boxes[20].x; + // int y1_1 = dd_1.boxes[20].y; + // int x2_1 = x1_1 + dd_1.boxes[20].width(); + // int y2_1 = y2_1 + dd_1.boxes[20].height(); + // float confidence_1 = dd_1.confidences[20]; + // int classId_1 = dd_1.classIds[20]; + + // int x1_2 = dd_2.boxes[20].x; + // int y1_2 = dd_2.boxes[20].y; + // int x2_2 = x1_2 + dd_2.boxes[20].width(); + // int y2_2 = y2_2 + dd_2.boxes[20].height(); + // float confidence_2 = dd_2.confidences[20]; + // int classId_2 = dd_2.classIds[20]; + + } + +} // SUITE(Frame_Tests)