about Tracking #907

jiamingzhang94 · 2018-05-25T15:18:18Z

Hi, @AlexeyAB
I want to implement tracking by using Yolo v2 or v3.
Just want to mark the midpoint of the detected object in each frame of the video, via opencv or other tools.
Then connect the points in two frames to form a trajectory curve.
I know this idea is very simple, but given my poor programming skills, I can't find this function or the API I want from numerous .c files.
So can you tell me what should I do? Or give me some guidance to look for those function?
Thank you very much!

AlexeyAB · 2018-05-25T18:11:29Z

@z490938304 Hi,

Un-comment this line:

darknet/src/yolo_console_dll.cpp

Line 18 in 6390a5a

//#define TRACK_OPTFLOW
Compile with LIBSO=1 in the Makefile on Linux, and run:
LD_LIBRARY_PATH=./:$LD_LIBRARY_PATH ./uselib data/coco.names yolov3.cfg yolov3.weights test.mp4
- Or on Windows compile yolo_cpp_dll.sln and then compile yolo_console_dll.sln and run yolo_console_dll.exe data/coco.names yolov3.cfg yolov3.weights test.mp4

It will detect and track objects on videofile.

If you want to track large objects, then change win_size = 9:

darknet/src/yolo_v2_class.hpp

Line 162 in 6390a5a

    
           Tracker_optflow(int _gpu_id = 0, int win_size = 9, int max_level = 3, int iterations = 8000, int _flow_error = -1) :

to this: win_size = 31

Code of multiple object tracking:

darknet/src/yolo_v2_class.hpp

Lines 155 to 322 in 6390a5a

    
           class Tracker_optflow { 
        
           public: 
        
           	const int gpu_count; 
        
           	const int gpu_id; 
        
           	const int flow_error; 
        
           	Tracker_optflow(int _gpu_id = 0, int win_size = 9, int max_level = 3, int iterations = 8000, int _flow_error = -1) : 
        
           		gpu_count(cv::cuda::getCudaEnabledDeviceCount()), gpu_id(std::min(_gpu_id, gpu_count-1)), 
        
           		flow_error((_flow_error > 0)? _flow_error:(win_size*4)) 
        
           	{ 
        
           		int const old_gpu_id = cv::cuda::getDevice(); 
        
           		cv::cuda::setDevice(gpu_id); 
        
           		stream = cv::cuda::Stream(); 
        
           		sync_PyrLKOpticalFlow_gpu = cv::cuda::SparsePyrLKOpticalFlow::create(); 
        
           		sync_PyrLKOpticalFlow_gpu->setWinSize(cv::Size(win_size, win_size));	// 9, 15, 21, 31 
        
           		sync_PyrLKOpticalFlow_gpu->setMaxLevel(max_level);		// +- 3 pt 
        
           		sync_PyrLKOpticalFlow_gpu->setNumIters(iterations);	// 2000, def: 30 
        
           		cv::cuda::setDevice(old_gpu_id); 
        
           	} 
        
           	// just to avoid extra allocations 
        
           	cv::cuda::GpuMat src_mat_gpu; 
        
           	cv::cuda::GpuMat dst_mat_gpu, dst_grey_gpu; 
        
           	cv::cuda::GpuMat prev_pts_flow_gpu, cur_pts_flow_gpu; 
        
           	cv::cuda::GpuMat status_gpu, err_gpu; 
        
           	cv::cuda::GpuMat src_grey_gpu;	// used in both functions 
        
           	cv::Ptr<cv::cuda::SparsePyrLKOpticalFlow> sync_PyrLKOpticalFlow_gpu; 
        
           	cv::cuda::Stream stream; 
        
           	std::vector<bbox_t> cur_bbox_vec; 
        
           	std::vector<bool> good_bbox_vec_flags; 
        
           	cv::Mat prev_pts_flow_cpu; 
        
           	void update_cur_bbox_vec(std::vector<bbox_t> _cur_bbox_vec) 
        
           	{ 
        
           		cur_bbox_vec = _cur_bbox_vec; 
        
           		good_bbox_vec_flags = std::vector<bool>(cur_bbox_vec.size(), true); 
        
           		cv::Mat prev_pts, cur_pts_flow_cpu; 
        
           		for (auto &i : cur_bbox_vec) { 
        
           			float x_center = (i.x + i.w / 2.0F); 
        
           			float y_center = (i.y + i.h / 2.0F); 
        
           			prev_pts.push_back(cv::Point2f(x_center, y_center)); 
        
           		} 
        
           		if (prev_pts.rows == 0) 
        
           			prev_pts_flow_cpu = cv::Mat(); 
        
           		else 
        
           			cv::transpose(prev_pts, prev_pts_flow_cpu); 
        
           		if (prev_pts_flow_gpu.cols < prev_pts_flow_cpu.cols) { 
        
           			prev_pts_flow_gpu = cv::cuda::GpuMat(prev_pts_flow_cpu.size(), prev_pts_flow_cpu.type()); 
        
           			cur_pts_flow_gpu = cv::cuda::GpuMat(prev_pts_flow_cpu.size(), prev_pts_flow_cpu.type()); 
        
           			status_gpu = cv::cuda::GpuMat(prev_pts_flow_cpu.size(), CV_8UC1); 
        
           			err_gpu = cv::cuda::GpuMat(prev_pts_flow_cpu.size(), CV_32FC1); 
        
           		} 
        
           		prev_pts_flow_gpu.upload(cv::Mat(prev_pts_flow_cpu), stream); 
        
           	} 
        
           	void update_tracking_flow(cv::Mat src_mat, std::vector<bbox_t> _cur_bbox_vec) 
        
           	{ 
        
           		int const old_gpu_id = cv::cuda::getDevice(); 
        
           		if (old_gpu_id != gpu_id) 
        
           			cv::cuda::setDevice(gpu_id); 
        
           		if (src_mat.channels() == 3) { 
        
           			if (src_mat_gpu.cols == 0) { 
        
           				src_mat_gpu = cv::cuda::GpuMat(src_mat.size(), src_mat.type()); 
        
           				src_grey_gpu = cv::cuda::GpuMat(src_mat.size(), CV_8UC1); 
        
           			} 
        
           			update_cur_bbox_vec(_cur_bbox_vec); 
        
           			//src_grey_gpu.upload(src_mat, stream);	// use BGR 
        
           			src_mat_gpu.upload(src_mat, stream); 
        
           			cv::cuda::cvtColor(src_mat_gpu, src_grey_gpu, CV_BGR2GRAY, 1, stream); 
        
           		} 
        
           		if (old_gpu_id != gpu_id) 
        
           			cv::cuda::setDevice(old_gpu_id); 
        
           	} 
        
           	std::vector<bbox_t> tracking_flow(cv::Mat dst_mat, bool check_error = true) 
        
           	{ 
        
           		if (sync_PyrLKOpticalFlow_gpu.empty()) { 
        
           			std::cout << "sync_PyrLKOpticalFlow_gpu isn't initialized \n"; 
        
           			return cur_bbox_vec; 
        
           		} 
        
           		int const old_gpu_id = cv::cuda::getDevice(); 
        
           		if(old_gpu_id != gpu_id) 
        
           			cv::cuda::setDevice(gpu_id); 
        
           		if (dst_mat_gpu.cols == 0) { 
        
           			dst_mat_gpu = cv::cuda::GpuMat(dst_mat.size(), dst_mat.type()); 
        
           			dst_grey_gpu = cv::cuda::GpuMat(dst_mat.size(), CV_8UC1); 
        
           		} 
        
           		//dst_grey_gpu.upload(dst_mat, stream);	// use BGR 
        
           		dst_mat_gpu.upload(dst_mat, stream); 
        
           		cv::cuda::cvtColor(dst_mat_gpu, dst_grey_gpu, CV_BGR2GRAY, 1, stream); 
        
           		if (src_grey_gpu.rows != dst_grey_gpu.rows || src_grey_gpu.cols != dst_grey_gpu.cols) { 
        
           			stream.waitForCompletion(); 
        
           			src_grey_gpu = dst_grey_gpu.clone(); 
        
           			cv::cuda::setDevice(old_gpu_id); 
        
           			return cur_bbox_vec; 
        
           		} 
        
           		////sync_PyrLKOpticalFlow_gpu.sparse(src_grey_gpu, dst_grey_gpu, prev_pts_flow_gpu, cur_pts_flow_gpu, status_gpu, &err_gpu);	// OpenCV 2.4.x 
        
           		sync_PyrLKOpticalFlow_gpu->calc(src_grey_gpu, dst_grey_gpu, prev_pts_flow_gpu, cur_pts_flow_gpu, status_gpu, err_gpu, stream);	// OpenCV 3.x 
        
           		cv::Mat cur_pts_flow_cpu; 
        
           		cur_pts_flow_gpu.download(cur_pts_flow_cpu, stream); 
        
           		dst_grey_gpu.copyTo(src_grey_gpu, stream); 
        
           		cv::Mat err_cpu, status_cpu; 
        
           		err_gpu.download(err_cpu, stream); 
        
           		status_gpu.download(status_cpu, stream); 
        
           		stream.waitForCompletion(); 
        
           		std::vector<bbox_t> result_bbox_vec; 
        
           		if (err_cpu.cols == cur_bbox_vec.size() && status_cpu.cols == cur_bbox_vec.size())  
        
           		{ 
        
           			for (size_t i = 0; i < cur_bbox_vec.size(); ++i) 
        
           			{ 
        
           				cv::Point2f cur_key_pt = cur_pts_flow_cpu.at<cv::Point2f>(0, i); 
        
           				cv::Point2f prev_key_pt = prev_pts_flow_cpu.at<cv::Point2f>(0, i); 
        
           				float moved_x = cur_key_pt.x - prev_key_pt.x; 
        
           				float moved_y = cur_key_pt.y - prev_key_pt.y; 
        
           				if (abs(moved_x) < 100 && abs(moved_y) < 100 && good_bbox_vec_flags[i]) 
        
           					if (err_cpu.at<float>(0, i) < flow_error && status_cpu.at<unsigned char>(0, i) != 0 && 
        
           						((float)cur_bbox_vec[i].x + moved_x) > 0 && ((float)cur_bbox_vec[i].y + moved_y) > 0) 
        
           					{ 
        
           						cur_bbox_vec[i].x += moved_x + 0.5; 
        
           						cur_bbox_vec[i].y += moved_y + 0.5; 
        
           						result_bbox_vec.push_back(cur_bbox_vec[i]); 
        
           					} 
        
           					else good_bbox_vec_flags[i] = false; 
        
           				else good_bbox_vec_flags[i] = false; 
        
           				//if(!check_error && !good_bbox_vec_flags[i]) result_bbox_vec.push_back(cur_bbox_vec[i]); 
        
           			} 
        
           		} 
        
           		cur_pts_flow_gpu.swap(prev_pts_flow_gpu); 
        
           		cur_pts_flow_cpu.copyTo(prev_pts_flow_cpu); 
        
           		if (old_gpu_id != gpu_id) 
        
           			cv::cuda::setDevice(old_gpu_id); 
        
           		return result_bbox_vec; 
        
           	} 
        
           };

uday60 · 2018-05-25T21:38:47Z

@AlexeyAB

I am getting this error : cannot find this file opencv_cudaoptflow340.lib ...

AlexeyAB · 2018-05-25T21:48:08Z

@uday60

Or you should compile OpenCV with CUDA

cmake -D CMAKE_BUILD_TYPE=RELEASE -D BUILD_EXAMPLES=ON -D WITH_FFMPEG=ON -D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUDA=ON -D CUDA_ARCH_BIN="3.0 5.0 6.1" -D WITH_CUBLAS=ON -D WITH_CUFFT=ON -D WITH_EIGEN=ON ./

make all -j8
make install

Or you should change this line:

darknet/src/yolo_v2_class.hpp

Line 148 in 6390a5a

#if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(GPU)

to this
#if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(NOTUSED)

jiamingzhang94 · 2018-05-26T05:11:04Z

Thanks for your replying!
I am meeting the same question as @uday60 .
And I have tired the method 2, but it doesn't work.
I am trying compile opencv with CUDA on Windows

AlexeyAB · 2018-05-26T09:48:29Z

@z490938304

And I have tired the method 2, but it doesn't work.

What error do you get?

jiamingzhang94 · 2018-05-26T10:08:23Z

@AlexeyAB
cannot find this file opencv_cudaoptflow340.lib
And I tried to change this line
#if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(GPU)
to
#if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(NOTUSED)
But it doesn't work

uday60 · 2018-05-26T21:26:37Z

So here are the steps to run it on windows given in this article:
https://jamesbowley.co.uk/build-opencv-v3-2-with-cuda-8-0-and-intel-mkltbb/
I was able to do it on opencv 3.4.0

Many lib files will be generated in the release folder but we need the following:

After this copy the following dll files from bin/release folder

@AlexeyAB -out_filename parameter is not supported here right ?

jiamingzhang94 · 2018-05-28T05:40:15Z

@AlexeyAB
I think this is my last question before the no error...
I have built the yolo_cpp_dll.exe .

But appear 'Used GPU 0' and 'OpenCV Error: No CUDA support (The library is compiled without CUDA support) in throw_no_cuda, file C:\build\master_winpack-build-win64-vc14\opencv\modules\core\include\opencv2/core/private.cuda.hpp, line 107'

But I have complied opencv with cuda. I think the key is my GPU 0 is Core Graphics. My Nvida GPU is GPU 1.
How can I change the setting to use GPU 1?

AlexeyAB · 2018-05-28T10:17:07Z

@z490938304 No, it means that you OpenCV is compiled without CUDA.

Change this line:

darknet/src/yolo_v2_class.hpp

Line 148 in 6390a5a

#if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(GPU)

to this:
#if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(NOTUSED)

And show screenshot of error.

Choneke · 2018-06-04T03:35:35Z

Get the same error message when doing Build->Build yolo_console_dll as you said for detecting and tracking in video file

I try to look for that file in C:\opencv_3.0\opencv\build\x64\vc14\lib, but no file with name opencv_cudaoptflow340.lib

I tried all the methods (1) and (2) which mentioned above.

Or on Windows compile yolo_cpp_dll.sln and then compile yolo_console_dll.sln and run yolo_console_dll.exe data/coco.names yolov3.cfg yolov3.weights test.mp4
It will detect and track objects on videofile

AlexeyAB · 2018-06-04T10:31:08Z

@Choneke

Comment these lines:

darknet/src/yolo_console_dll.cpp

Lines 34 to 38 in 132251d

    
           #pragma comment(lib, "opencv_cudaoptflow" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_cudaimgproc" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_core" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_imgproc" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_highgui" OPENCV_VERSION ".lib")

Change this line:

darknet/src/yolo_v2_class.hpp

Line 148 in 6390a5a

#if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(GPU)

to this: #if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(NOTUSED)

Isha8 · 2018-06-27T17:27:05Z

@AlexeyAB Is there a way to implement a function such that this -> #954 (comment) can be used along with the object tracking of this thread, to save the tracked labels once rather than save the same label multiple times depending on the fps of the video feed.

JCwww · 2018-09-16T09:17:10Z

@z490938304 Did you make it? connect the mid points

buzdarbalooch · 2018-11-21T11:56:19Z

@AlexeyAB

dear AlexeyAB after adding these lines #if defined(TRACK_OPTFLOW) && defined(OPENCV) && defined(NOTUSED). and uncommenting this line //#define TRACK_OPTFLOW .

When i do make it gives me this error.

buzdarbalooch · 2018-11-21T12:06:00Z

secondly to see the results in console (linux) one should execute this

./darknet detector test cfg/obj.data cfg/yolo.cfg backup/yolo_10100.weights data/5.jpg

OR this

LD_LIBRARY_PATH=./:$LD_LIBRARY_PATH ./uselib data/coco.names yolov3.cfg yolov3.weights test.mp4

Can we see the result on images also or only videos. Eg if i give the i/p to the system like this image. where there are multiple objects. Will the system put an id on every object.

hadign20 · 2019-03-13T19:50:49Z

So here are the steps to run it on windows given in this article:
https://jamesbowley.co.uk/build-opencv-v3-2-with-cuda-8-0-and-intel-mkltbb/
I was able to do it on opencv 3.4.0

Many lib files will be generated in the release folder but we need the following:

After this copy the following dll files from bin/release folder

@AlexeyAB -out_filename parameter is not supported here right ?

I do not have opencv_cudaoptflow340.lib and some other libraries in C:\opencv_3.4.0\opencv\build\lib\Release. I compiled opencv 3.4.0 with cuda9.1 using cmake. I only have these:

AlexeyAB · 2019-03-13T21:42:23Z

@hadi-ghnd Hi,

Try to change this line:

darknet/src/yolo_console_dll.cpp

Line 14 in 7a85430

//#define TRACK_OPTFLOW

to these

#define TRACK_OPTFLOW
#define GPU

And comment these lines:

darknet/src/yolo_console_dll.cpp

Lines 31 to 35 in 7a85430

    
           #pragma comment(lib, "opencv_cudaoptflow" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_cudaimgproc" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_core" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_imgproc" OPENCV_VERSION ".lib") 
        
           #pragma comment(lib, "opencv_highgui" OPENCV_VERSION ".lib")

Then try to compile build/darknet/yolo_cpp_dll.sln and build/darknet/yolo_console_dll.sln - does it work?

sctrueew · 2020-07-18T11:37:43Z

@AlexeyAB Hi,

I have a video with 5 FPS. What parameters should I change for this case in Tracker_optflow?

Thanks

Choneke mentioned this issue Jun 4, 2018

Cannot open file 'opencv_cudaoptflow340.lib' #976

Open

dexception mentioned this issue Jun 12, 2018

Dlib Face Recognition + Yolov3 tracker #1030

Open

AlexeyAB mentioned this issue Jun 27, 2018

Tracking in Yolov3 #1105

Open

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

about Tracking #907

about Tracking #907

jiamingzhang94 commented May 25, 2018 •

edited

Loading

AlexeyAB commented May 25, 2018

uday60 commented May 25, 2018

AlexeyAB commented May 25, 2018

jiamingzhang94 commented May 26, 2018

AlexeyAB commented May 26, 2018

jiamingzhang94 commented May 26, 2018

uday60 commented May 26, 2018 •

edited

Loading

jiamingzhang94 commented May 28, 2018

AlexeyAB commented May 28, 2018

Choneke commented Jun 4, 2018 •

edited

Loading

AlexeyAB commented Jun 4, 2018

Isha8 commented Jun 27, 2018 •

edited

Loading

JCwww commented Sep 16, 2018

buzdarbalooch commented Nov 21, 2018

buzdarbalooch commented Nov 21, 2018

hadign20 commented Mar 13, 2019

AlexeyAB commented Mar 13, 2019

sctrueew commented Jul 18, 2020

about Tracking #907

about Tracking #907

Comments

jiamingzhang94 commented May 25, 2018 • edited Loading

AlexeyAB commented May 25, 2018

uday60 commented May 25, 2018

AlexeyAB commented May 25, 2018

jiamingzhang94 commented May 26, 2018

AlexeyAB commented May 26, 2018

jiamingzhang94 commented May 26, 2018

uday60 commented May 26, 2018 • edited Loading

jiamingzhang94 commented May 28, 2018

AlexeyAB commented May 28, 2018

Choneke commented Jun 4, 2018 • edited Loading

AlexeyAB commented Jun 4, 2018

Isha8 commented Jun 27, 2018 • edited Loading

JCwww commented Sep 16, 2018

buzdarbalooch commented Nov 21, 2018

buzdarbalooch commented Nov 21, 2018

hadign20 commented Mar 13, 2019

AlexeyAB commented Mar 13, 2019

sctrueew commented Jul 18, 2020

jiamingzhang94 commented May 25, 2018 •

edited

Loading

uday60 commented May 26, 2018 •

edited

Loading

Choneke commented Jun 4, 2018 •

edited

Loading

Isha8 commented Jun 27, 2018 •

edited

Loading