Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Add a new feature: automatically detecting an object using background…

… subtraction. Now the object does not need to be present in the first frame. However, a background image is required.
  • Loading branch information...
commit 02ce59101a2cc493ab6795a2a1ed5b10c8e6f5f4 1 parent f5e0c6e
@TheMrtN authored
View
50 run_TLD.m
@@ -25,9 +25,11 @@
% Feature flags
remember_fps = 0; % toggle remembering the fps in a global vector called 'fps'
save_object = 0; % toggle saving the picture inside the bounding box of every frame
+detect_object = 0; % toggle detecting if an object appears in the scene
% Input / output settings
-opt.source = struct('camera',0,'input','_input/','bb0',[]); % camera/directory swith, directory_name, initial_bounding_box (if empty, it will be selected by the user)
+input = '_input/'; % motorbike example
+opt.source = struct('camera',0,'input',input,'bb0',[]); % camera/directory swith, directory_name, initial_bounding_box (if empty, it will be selected by the user)
opt.output = '_output/'; mkdir(opt.output); % output directory that will contain bounding boxes + confidence
if save_object
opt.object = '_object/'; mkdir(opt.object); delete([opt.object '*.png']); % output directory that will contain the images inside the bounding box of every frame
@@ -41,14 +43,15 @@
update_detector = 1; % online learning on/off, of 0 detector is trained only in the first frame and then remains fixed
opt.plot = struct('pex',1,'nex',1,'dt',1,'confidence',1,'target',1,'replace',0,'drawoutput',3,'draw',0,'pts',1,'help', 0,'patch_rescale',1,'save',0,'save_object',save_object);
-% If we don't use camera input, the init.txt does not exist and there is a
-% background directory, determine the bounding box of the input by making a
-% init.txt with the bounding box of the biggest blob in the first frame.
+% If we don't wait for the object to appear, we don't use camera input, the
+% init.txt does not exist and there is a background directory, determine
+% the bounding box of the input by making a init.txt with the bounding box
+% of the biggest blob in the first frame.
% This requires a picture of the background in the folder [opt.source.input
% 'background/'].
-if ~opt.source.camera && ~exist([opt.source.input 'init.txt'], 'file') && exist([opt.source.input 'background/'], 'dir')
+if ~detect_object && ~opt.source.camera && ~exist([opt.source.input 'init.txt'], 'file') && exist([opt.source.input 'background/'], 'dir')
determine_initial_bb(opt.source.input, min_win);
-end; % else, the bounding box is determined by the user.
+end; % else, the bounding box is determined by the user or the existing init.txt is used.
% Do-not-change -----------------------------------------------------------
@@ -67,10 +70,37 @@
fps = [];
end;
-%profile on;
-[bb,conf] = tldExample(opt);
-%profile off;
-%profile viewer;
+% If detect_object is disabled, run TLD as usual.
+bb = []; conf = [];
+if ~detect_object
+ %profile on;
+ [bb,conf] = tldExample(opt);
+ %profile off;
+ %profile viewer;
+else % If detect_object is enabled.
+ % Play the frame sequence until an object of sufficient size is
+ % detected.
+ [i, initialBB] = tldWaitForObject(opt);
+ % If it returned a valid index and bounding box, run TLD.
+ if i ~= -1 && ~isempty(initialBB)
+ % Save the handle, but clear the rest of the tld variable.
+ global tld;
+ handle = -1;
+ if isfield(tld,'handle')
+ handle = tld.handle;
+ end;
+ clear global tld;
+
+ % Run TLD.
+ [bb, conf] = tldExample2(opt, i, initialBB, handle);
+ elseif i == 1 && isempty(initialBB) % Something went wrong during initialization.
+ % Run TLD as usual.
+ disp(['Notification from ' mfilename ': intialization of automatically detecting an object failed. Starting TLD as usual.']);
+ [bb,conf] = tldExample(opt);
+ else
+ disp(['Notification from ' mfilename ': waited for an object to appear, but reached the end of the frame sequence before we could find one.']);
+ end;
+end;
% Save results ------------------------------------------------------------
dlmwrite([opt.output '/tld.txt'],[bb; conf]');
View
71 tld/tldExample2.m
@@ -0,0 +1,71 @@
+% Copyright 2011 Zdekek Kalal
+% File created by Maarten Somhorst.
+%
+% This file is not part of the original TLD. However, most of the lines are
+% copied from tldExample or other functions (mentioned in the comments).
+%
+% TLD is free software: you can redistribute it and/or modify
+% it under the terms of the GNU General Public License as published by
+% the Free Software Foundation, either version 3 of the License, or
+% (at your option) any later version.
+%
+% TLD is distributed in the hope that it will be useful,
+% but WITHOUT ANY WARRANTY; without even the implied warranty of
+% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+% GNU General Public License for more details.
+%
+% You should have received a copy of the GNU General Public License
+% along with TLD. If not, see <http://www.gnu.org/licenses/>.
+%
+% tldExample2(opt, startIndex, bb, handle) works a lot like tldExample(opt),
+% but is able to start at the frame determined by the startIndex parameter.
+% Furthermore, an existing handle can be given as a parameter.
+
+function [bb, conf] = tldExample2(opt, startIndex, initialBB, handle)
+
+% Check index.
+if startIndex <= 0
+ disp(['Warning in ' mfilename ': start index cannot be smaller than or equal to zero. An index of one is assumed.']);
+ startIndex = 1;
+end;
+% Check initialBB.
+if size(initialBB,1) ~= 4
+ disp(['Error in ' mfilename ': intial bounding box has a unusual size. Cannot initialize TLD.']);
+ bb = []; conf = [];
+ return;
+end;
+
+global tld; % holds results and temporal variables
+
+% If a valid handle is given, save it.
+if handle ~= -1
+ tld.handle = handle;
+end;
+
+% INITIALIZATION ----------------------------------------------------------
+
+% From: tldInitSource
+opt.source.files = img_dir(opt.source.input);
+opt.source.files = opt.source.files(startIndex:end); % Filter the relevant frames.
+opt.source.idx = 1:length(opt.source.files);
+% -------------------
+
+figure(2);
+
+% From: tldInitFirstFrame
+opt.source.im0 = img_get(opt.source, opt.source.idx(1));
+opt.source.bb = initialBB;
+% -----------------------
+
+tld = tldInit(opt,tld); % train initial detector and initialize the 'tld' structure
+tld = tldDisplay(0,tld); % initialize display
+
+% RUN-TIME ----------------------------------------------------------------
+
+tld = tldExampleLoop(tld);
+
+% RETURN RESULTS ----------------------------------------------------------
+
+bb = tld.bb; conf = tld.conf;
+
+end
View
124 tld/tldWaitForObject.m
@@ -0,0 +1,124 @@
+% Copyright 2011 Zdekek Kalal
+% File created by Maarten Somhorst
+%
+% This file is not part of the original TLD. However, most of the lines are
+% copied from tldExample or other functions (mentioned in the comments).
+%
+% TLD is free software: you can redistribute it and/or modify
+% it under the terms of the GNU General Public License as published by
+% the Free Software Foundation, either version 3 of the License, or
+% (at your option) any later version.
+%
+% TLD is distributed in the hope that it will be useful,
+% but WITHOUT ANY WARRANTY; without even the implied warranty of
+% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+% GNU General Public License for more details.
+%
+% You should have received a copy of the GNU General Public License
+% along with TLD. If not, see <http://www.gnu.org/licenses/>.
+%
+% tldWaitForObject(opt) processes and displays each frame until an object
+% is detect. Then it returns the frame index and the bounding box.
+%
+% If no object is found and the last frame is processed, an index of -1 and
+% an empty bounding box is returned. If something went wrong with the
+% background image, the index is set to 1 and an empty bounding box is
+% returned.
+
+function [index, bb] = tldWaitForObject(opt)
+
+% INITIALIZATION ----------------------------------------------------------
+index = -1;
+bb = [];
+
+global tld;
+opt.source = tldInitSource(opt.source);
+
+figure(2); set(2,'KeyPressFcn', @handleKey); % open figure for display of results
+finish = 0;
+function handleKey(dummy1,dummy2), finish = 1; fprintf('Execution interrupted by keypress.\n'); end % by pressing any key, the process will exit
+
+% From: tldInitFirstFrame
+opt.source.im0 = img_get(opt.source,opt.source.idx(1));
+% -----------------------
+
+% From: tldInit
+if ~isempty(tld);
+ handle = tld.handle;
+ tld = opt;
+ tld.handle = handle;
+else
+ tld = opt;
+end
+
+tld.img = cell(1,length(tld.source.idx));
+tld.img{1} = tld.source.im0;
+% -------------
+
+% Load background picture.
+bg = get_background_image(tld.source.input);
+if isempty(bg)
+ disp(['Error in ' mfilename ': no background image found. Returning.']);
+ index = 1;
+ return;
+end;
+
+% Check if the first frame already contains the object. If so, return.
+bb = get_bb_backgroundsubtraction(bg, tld.img{1}.input);
+if ~isempty(bb) && min(bb_size(bb)) >= tld.model.min_win
+ index = 1;
+ return;
+end;
+
+% From: tldDisplay
+tld.handle = imshow(tld.img{1}.input,'initialmagnification','fit');
+set(gcf,'MenuBar','none','ToolBar','none','color',[0 0 0]);
+set(gca,'position',[0 0 1 1]);
+set(tld.handle,'cdata',tld.img{1}.input);
+hold on;
+set(gcf,'Position',[100 100 [640 360]]);
+% ----------------
+
+
+% RUN-TIME ----------------------------------------------------------------
+
+for i = 2:length(tld.source.idx) % for every frame
+
+ % From: tldProcessFrame
+ tld.img{i} = img_get(tld.source,i); % grab frame
+ % ---------------------
+
+ % Check if this frame contains an object. If so, return.
+ bb = get_bb_backgroundsubtraction(bg, tld.img{i}.input);
+ if ~isempty(bb) && min(bb_size(bb)) >= tld.model.min_win
+ index = i;
+ return;
+ end;
+
+ % From: tldDisplay
+ h = get(gca,'Children'); delete(h(1:end-1));
+ img = tld.img{i}.input; % draw image
+ set(tld.handle,'cdata',img); hold on;
+ drawnow;
+ tic;
+ % ----------------
+
+ if finish % finish if any key was pressed
+ if tld.source.camera
+ stoppreview(tld.source.vid);
+ closepreview(tld.source.vid);
+ close(1);
+ end
+ close(2);
+ return;
+ end
+
+ if tld.plot.save == 1
+ img = getframe;
+ imwrite(img.cdata,[tld.output num2str(i,'%05d') '.png']);
+ end
+
+end
+
+end
+
View
104 utils/determine_initial_bb.m
@@ -8,7 +8,8 @@ function determine_initial_bb(input, min_win)
% 'input_directory/background/'. This function substracts the background
% from the first frame in order to find the largest blob in the picture.
% The bounding box coordinates are written to 'input_directory/init.txt'
-% so that TLD can use it.
+% so that TLD can use it. For this substraction process, the function
+% get_bb_backgroundsubstraction() is used.
%
% When the bounding box of the object is determined, this function checks
% if the size (minimum of width and height) is smaller than min_win. If
@@ -21,41 +22,12 @@ function determine_initial_bb(input, min_win)
% Copyright 2011 by Maarten Somhorst.
%% Constants.
-% The connectivity used for the command bwlabel. Possible values: 4 or 8.
-connectivity = 4;
-% The maximum number of gray levels a pixel may differ to be excluded from
-% the binary picture. A value of zero includes all pixels from the first
-% frame that have a different gray value at the background. A value of one
-% allows a pixel to have a gray level +1 or -1, and so on.
-diffThresold = 10;
-% The minimum fraction of pixels an object must have to be handled as an
-% object. Minimum is 0, maximum is 1.
-blobSizeThresold = 0.0;
% The string where each error ends with if it exits this function.
returnString = 'Unable to determine bounding box.';
-% Demo flag.
-demo = false;
-%% Load pics
-% Check if there's a background picture.
-bgDir = [input 'background/'];
-if(~exist(bgDir,'dir'))
- disp(['Error in ' mfilename ': the folder ' bgDir ' does not exist. ' returnString]);
- return
-end;
-bgFile = img_dir(bgDir);
-numBgFiles = length(bgFile);
-if(numBgFiles == 0)
- disp(['Error in ' mfilename ': there are no images in the folder ' bgDir '. Please provide one image of the background. ' returnString]);
- return
-elseif(numBgFiles > 1)
- disp(['Warning from ' mfilename ': the folder ' bgDir ' contains ' int2str(numBgFiles) ' pictures. The first one is used.']);
-end;
-% Load the background picture.
-bgPic = imread(bgFile(1).name);
-if(ndims(bgPic)==3)
- bgPic = rgb2gray(bgPic);
-end;
+%% Load pictures.
+% Get background picture.
+bgPic = get_background_image(input);
% Check if there is data.
dataFiles = img_dir(input);
@@ -69,66 +41,18 @@ function determine_initial_bb(input, min_win)
firstFrame = rgb2gray(firstFrame);
end;
-%% Calculate difference between background and first frame
-diffGray = abs(bgPic - firstFrame);
-% Make it binary.
-diffBinary = diffGray > diffThresold;
-% If demo=true, show it.
-if demo; figure(8); imshow(diffBinary); end;
-
-%% Find the biggest blob.
-% Label all connected blobs.
-[labeledObjects, numObjects] = bwlabel(diffBinary, connectivity);
-
-% Count the number pixels per blob.
-[u, m, n] = unique(labeledObjects);
-counts = accumarray(n(:), 1);
-
-% Remove the one that count the background pixels.
-background = find(u==0);
-u(background) = [];
-counts(background) = [];
-
-% Determine the biggest blob and check if it is large enough to be an
-% object.
-[y,indexes]=max(counts);
-[rows,cols] = size(diffBinary);
-fractionCovered = y/(rows*cols);
-if demo
- disp(['Number of objects found: ' int2str(numObjects) '.']);
- disp(['The biggest object has ' int2str(y) ' pixels (' sprintf('%0.1f', fractionCovered*100) '%) and id ' int2str(u(indexes)) '.']);
-end;
-if fractionCovered < blobSizeThresold
- disp(['Warning from ' mfilename ': no object found that covers ' sprintf('%0.1f', blobSizeThresold*100) '% of the first frame or more. ' returnString]);
- return
-end;
-
-% Make a new image with that single blob and show it.
-filteredImage = (labeledObjects==u(indexes))>0;
-if demo; figure(9); imshow(filteredImage); end;
-
-% Find the pixels of that single blob.
-[i,j] = find(filteredImage>0);
+%% Get bounding box.
+coordsVector = get_bb_backgroundsubtraction(bgPic, firstFrame);
-%% Determine the bounding box and write it to init.txt.
-% First we determine some correction variables to correct for shadows.
-minI = min(i);
-maxI = max(i);
-blobHeight = maxI - minI;
-minJ = min(j);
-maxJ = max(j);
-blobWidth = maxJ - minJ;
-correctionSides = .1 * blobWidth; % 10% correction for the sides.
-correctionTop = .05 * blobHeight; % 5% correction for the top.
-correctionBottom = .1 * blobHeight; % 10% correction for the bottom.
-
-% Create the vector, check it and write it to init.txt.
-coordsVector = [round(minJ+correctionSides) round(minI+correctionTop) round(maxJ-correctionSides) round(maxI-correctionBottom)];
-if min(bb_size(coordsVector')) < min_win
+%% Check the size of the bounding box and save it when OK.
+if isempty(coordsVector)
+ disp(['Warning from ' mfilename ': the size of the object is smaller than the threshold. ' returnString]);
+ return;
+elseif min(bb_size(coordsVector)) < min_win
% If the bounding box is too big, we print a warning and return.
disp(['Warning from ' mfilename ': the size of the determined bounding box is not large enough for TLD. ' returnString]);
- return
+ return;
end;
-dlmwrite([input 'init.txt'], coordsVector);
+dlmwrite([input 'init.txt'], coordsVector');
end
View
38 utils/get_background_image.m
@@ -0,0 +1,38 @@
+function [bgImage] = get_background_image(input)
+%GET_BACKGROUND_IMAGE Returns the background image.
+% This function returns the background image in the input folder, which
+% is expected in a folder called 'background'.
+%
+% Copyright 2011 by Maarten Somhorst.
+
+bgImage = [];
+
+%% Constants.
+% The string where each error ends with if it exits this function.
+returnString = 'Unable to get background image.';
+
+%% Check if there's a background folder.
+bgDir = [input 'background/'];
+if(~exist(bgDir,'dir'))
+ disp(['Error in ' mfilename ': the folder ' bgDir ' does not exist. ' returnString]);
+ return
+end;
+
+%% Check if there's exactly one picture in the background folder.
+bgFile = img_dir(bgDir);
+numBgFiles = length(bgFile);
+if(numBgFiles == 0)
+ disp(['Error in ' mfilename ': there are no images in the folder ' bgDir '. Please provide one image of the background. ' returnString]);
+ return
+elseif(numBgFiles > 1)
+ disp(['Warning from ' mfilename ': the folder ' bgDir ' contains ' int2str(numBgFiles) ' pictures. The first one is used.']);
+end;
+
+%% Read the background picture and convert to grayscale if needed.
+bgImage = imread(bgFile(1).name);
+if(ndims(bgImage)==3)
+ bgImage = rgb2gray(bgImage);
+end;
+
+end
+
View
93 utils/get_bb_backgroundsubtraction.m
@@ -0,0 +1,93 @@
+function [bb] = get_bb_backgroundsubtraction(bgImage, currentFrame)
+%GET_BB_BACKGROUNDSUBSTRACTION Determines the bounding box.
+% GET_BB_BACKGROUNDSUBSTRACTION(bgImage, currentFrame) determines the
+% bounding box of the object on currentFrame, by substracting the
+% background (bgImage) from it.
+%
+% Note that this function might not be very useful when the camera has
+% different positions when shooting the current frame and the background
+% picture.
+%
+% The contents of this function used to reside in the
+% determine_initial_bb function.
+%
+% Copyright 2011 by Maarten Somhorst.
+
+bb = [];
+
+%% Constants.
+% The connectivity used for the command bwlabel. Possible values: 4 or 8.
+connectivity = 4;
+% The maximum number of gray levels a pixel may differ to be excluded from
+% the binary picture. A value of zero includes all pixels from the first
+% frame that have a different gray value at the background. A value of one
+% allows a pixel to have a gray level +1 or -1, and so on.
+diffThresold = 10;
+% The minimum fraction of pixels an object must have to be handled as an
+% object. Minimum is 0, maximum is 1.
+blobSizeThresold = 0.005;
+% Demo flag.
+demo = false;
+
+%% Calculate difference between background and first frame
+diffGray = abs(bgImage - currentFrame);
+% Make it binary.
+diffBinary = diffGray > diffThresold;
+% If demo=true, show it.
+if demo; figure(8); imshow(diffBinary); end;
+
+%% Find the biggest blob.
+% Label all connected blobs.
+[labeledObjects, numObjects] = bwlabel(diffBinary, connectivity);
+
+% Count the number pixels per blob.
+[u, m, n] = unique(labeledObjects);
+counts = accumarray(n(:), 1);
+
+% Remove the one that count the background pixels.
+background = find(u==0);
+u(background) = [];
+counts(background) = [];
+
+% If u is empty, then both images were equal.
+if isempty(u)
+ return;
+end;
+
+% Determine the biggest blob and check if it is large enough to be an
+% object.
+[y,indexes]=max(counts);
+[rows,cols] = size(diffBinary);
+fractionCovered = y/(rows*cols);
+if demo
+ disp(['Number of objects found: ' int2str(numObjects) '.']);
+ disp(['The biggest object has ' int2str(y) ' pixels (' sprintf('%0.1f', fractionCovered*100) '%) and id ' int2str(u(indexes)) '.']);
+end;
+if fractionCovered < blobSizeThresold
+% disp(['Warning from ' mfilename ': no object found that covers ' sprintf('%0.1f', blobSizeThresold*100) '% of the first frame or more. ' ]); % returnString
+ return
+end;
+
+% Make a new image with that single blob and show it.
+filteredImage = (labeledObjects==u(indexes))>0;
+if demo; figure(9); imshow(filteredImage); end;
+
+% Find the pixels of that single blob.
+[i,j] = find(filteredImage>0);
+
+%% Determine the bounding box.
+% First we determine some correction variables to correct for shadows.
+minI = min(i);
+maxI = max(i);
+blobHeight = maxI - minI;
+minJ = min(j);
+maxJ = max(j);
+blobWidth = maxJ - minJ;
+correctionSides = .1 * blobWidth; % 10% correction for the sides.
+correctionTop = .05 * blobHeight; % 5% correction for the top.
+correctionBottom = .1 * blobHeight; % 10% correction for the bottom.
+
+% Create the bb vector.
+bb = [round(minJ+correctionSides) round(minI+correctionTop) round(maxJ-correctionSides) round(maxI-correctionBottom)]';
+
+end
Please sign in to comment.
Something went wrong with that request. Please try again.