Permalink
Switch branches/tags
Nothing to show
Find file
Fetching contributors…
Cannot retrieve contributors at this time
312 lines (255 sloc) 11.2 KB
function benchmarkDemo()
% BENCHMARKDEMO Demonstrates how to run the feature repatability benchmark
% Author: Karel Lenc and Andrea Vedaldi
% AUTORIGHTS
import localFeatures.*;
% --------------------------------------------------------------------
% PART 1: Detectors, visualization, and caching
% --------------------------------------------------------------------
% The feature detector/descriptor code is encapsualted in a corresponding.
% For example, VLFeatSift() encapslate the SIFT implementation in
% VLFeat.
%
% In addition to wrapping the detector code, each object instance
% contains a specific setting of parameters (for example, the
% cornerness threshold). In order to compare different parameter
% settings, one simply creates multiple instances of these objects.
siftDetector = VlFeatSift();
mserDetector = VlFeatMser('MinDiversity',0.5);
% VLBenchmarks enables a simple access to a number of public
% benchmakrs. It also provides simple facilities to generate test data
% on the fly. Here we generate an image consiting of a number of
% Gaussian blobs and we save it to disk for use with the detectors.
ellImage = datasets.helpers.genEllipticBlobs(...
'NumDeformations', 3,...
'Width', 400,...
'Height', 400);
tempPath = fileparts(tempname);
ellImagePath = fullfile(tempPath,'ellImage.png');
imwrite(ellImage,ellImagePath);
% Next, we extract the features by running the detectors we
% prepared.
%
% VLBeanchmarks is smart. The detector output is cached (for each
% input image and parameter setting), so the next time the detector is
% called the output is read from disk rather than being comptued
% again. VLBenchmarks automatically checks whether the detector
% parameters, image, or code change based on their modification date
% and invalidates the cache if necessary. You can also invoke the
% disableCaching() method in each detector to prevent it from caching.
siftFrames = siftDetector.extractFeatures(ellImagePath) ;
mserFrames = mserDetector.extractFeatures(ellImagePath) ;
% Now show the frames
figure(1); clf;
subplot(1,2,1); imshow(ellImage);
vl_plotframe(siftFrames,'LineWidth',1); title('SIFT frames');
subplot(1,2,2); imshow(ellImage);
vl_plotframe(mserFrames,'r','LineWidth',1); title('MSER frames');
delete(ellImagePath);
% --------------------------------------------------------------------
% PART 2: Detector repeatability
% --------------------------------------------------------------------
import datasets.*;
import benchmarks.*;
% A detector repeatability is measured against a benchmark. In this
% case we create an instance of the VGG Affine Testbed (graffity
% sequence).
dataset = VggAffineDataset('category','graf');
% Next, the benchmark is intialised by choosing various
% parameters. The defaults correspond to the seetting in the original
% publication (IJCV05).
repBenchmark = RepeatabilityBenchmark('Mode','Repeatability');
% Prepare three detectors, the two from PART 1 and a third one that
% simply detects features on a grid.
detectors{1} = siftDetector ;
detectors{2} = mserDetector ;
detectors{3} = ExampleLocalFeatureExtractor('Scales',5:5:25, ...
'FramesDistance',5) ;
% Now we are ready to run the repeatability test. We do this by fixing
% a reference image A and looping through other images B in the
% set. To this end we use the following information:
%
% dataset.NumImages:
% Number of images in the dataset.
%
% dataset.getImagePath(i):
% Path to the i-th image.
%
% dataset.getTransformation(i):
% Transformation from the first (reference) image to image i.
%
% Like for the detector output (see PART 1), VLBenchmarks caches the
% output of the test. This can be disabled by calling
% repBenchmark.disableCaching().
repeatability = [] ;
numCorresp = [] ;
for d = 1:numel(detectors)
for i = 2:dataset.NumImages
[repeatability(d,i) numCorresp(d,i)] = ...
repBenchmark.testFeatureExtractor(detectors{d}, ...
dataset.getTransformation(i), ...
dataset.getImagePath(1), ...
dataset.getImagePath(i)) ;
end
end
% The scores can now be prented, as well as visualized in a
% graph. This uses two simple functions defined below in this file.
detectorNames = {'SIFT','MSER','Features on a grid'};
printScores(detectorNames, 100 * repeatability, 'Repeatability');
printScores(detectorNames, numCorresp, 'Number of correspondences');
figure(2); clf;
subplot(1,2,1);
plotScores(detectorNames, dataset, 100 * repeatability, 'Repeatability');
subplot(1,2,2);
plotScores(detectorNames, dataset, numCorresp, 'Number of correspondences');
% Optionally, we can also see the matched frames itself. In this
% example we examine the matches between the reference and fourth
% image.
%
% We do this by running the repeatabiltiy score again. However, since
% the results are cached, this is fast.
imageBIdx = 4 ;
[drop drop siftCorresps siftReprojFrames] = ...
repBenchmark.testFeatureExtractor(siftDetector, ...
dataset.getTransformation(imageBIdx), ...
dataset.getImagePath(1), ...
dataset.getImagePath(imageBIdx)) ;
[drop drop mserCorresps mserReprojFrames] = ...
repBenchmark.testFeatureExtractor(mserDetector, ...
dataset.getTransformation(imageBIdx), ...
dataset.getImagePath(1), ...
dataset.getImagePath(imageBIdx)) ;
% And plot the feature frame correspondences
figure(3); clf;
image = imread(dataset.getImagePath(imageBIdx)) ;
subplot(1,2,1); imshow(image);
benchmarks.helpers.plotFrameMatches(siftCorresps, ...
siftReprojFrames,...
'IsReferenceImage',false);
title(sprintf('SIFT Correspondences with %d image (%s dataset).',...
imageBIdx,dataset.DatasetName));
subplot(1,2,2); imshow(image);
benchmarks.helpers.plotFrameMatches(mserCorresps, ...
mserReprojFrames,...
'IsReferenceImage',false);
title(sprintf('MSER Correspondences with %d image (%s dataset).',...
imageBIdx,dataset.DatasetName));
% --------------------------------------------------------------------
% PART 3: Detector matching score
% --------------------------------------------------------------------
% The matching score is similar to the repeatability score, but
% involves computing a descriptor. Detectors like SIFT bundle a
% descriptor as well. However, most of them (e.g. MSER) do not have an
% associated descriptor (e.g. MSER). In this case we can bind one of
% our choice by using the DescriptorAdapter class.
%
% In this particular example, the object encapsulating the SIFT
% detector is used as descriptor form MSER.
detectors{1} = siftDetector ;
detectors{2} = DescriptorAdapter(mserDetector,siftDetector) ;
% As an additional example, we show how to use a different descriptor
% for the SIFT detector. In this case, we bind the descriptor
% implemented by the ExampleLocalFeatureExtractor() class which simply
% computes the mean, standard deviation, and median of a patch.
%
% Note that in this manner the SIFT descriptor is replaced by the new
% descriptor.
meanVarMedianDescExtractor = ExampleLocalFeatureExtractor();
detectors{3} = DescriptorAdapter(siftDetector,meanVarMedianDescExtractor);
% We create a benchmark object and run the tests as before, but in
% this case we request that descriptor-based matched should be tested.
matchBenchmark = RepeatabilityBenchmark('Mode','MatchingScore');
matchScore = [] ;
numMatches = [] ;
for d = 1:numel(detectors)
for i = 2:dataset.NumImages
[matchScore(d,i) numMatches(d,i)] = ...
matchBenchmark.testFeatureExtractor(detectors{d}, ...
dataset.getTransformation(i), ...
dataset.getImagePath(1), ...
dataset.getImagePath(i)) ;
end
end
% Print and plot the results
detectorNames = {'SIFT', ...
'MSER + Sim. inv. SIFT desc.', ...
'SIFT frames + Mean/Var/Med desc.' };
printScores(detectorNames, matchScore*100, 'Match Score');
printScores(detectorNames, numMatches, 'Number of matches') ;
figure(4); clf;
subplot(1,2,1);
plotScores(detectorNames, dataset, matchScore*100,'Matching Score');
subplot(1,2,2);
plotScores(detectorNames, dataset, numMatches,'Number of matches');
% Same as with the correspondences, we can plot the matches based on
% feature frame descriptors. The code is nearly identical.
imageBIdx = 4 ;
[drop drop siftMatches siftReprojFrames] = ...
repBenchmark.testFeatureExtractor(siftDetector, ...
dataset.getTransformation(imageBIdx), ...
dataset.getImagePath(1), ...
dataset.getImagePath(imageBIdx)) ;
[drop drop mvmMatches mvmReprojFrames] = ...
repBenchmark.testFeatureExtractor(meanVarMedianDescExtractor, ...
dataset.getTransformation(imageBIdx), ...
dataset.getImagePath(1), ...
dataset.getImagePath(imageBIdx)) ;
figure(5); clf;
image = imread(dataset.getImagePath(imageBIdx));
subplot(1,2,1); imshow(image);
benchmarks.helpers.plotFrameMatches(siftMatches, siftReprojFrames,...
'IsReferenceImage',false);
title(sprintf('SIFT Matches with %d image (%s dataset).',...
imageBIdx,dataset.DatasetName));
subplot(1,2,2); imshow(image);
benchmarks.helpers.plotFrameMatches(mvmMatches, mvmReprojFrames,...
'IsReferenceImage',false);
title(sprintf('Matches using mean-variance-median descriptor with %d image (%s dataset).',...
imageBIdx,dataset.DatasetName));
% --------------------------------------------------------------------
% Helper functions
% --------------------------------------------------------------------
function printScores(detectorNames, scores, name)
numDetectors = numel(detectorNames);
maxNameLen = length('Method name');
for k = 1:numDetectors
maxNameLen = max(maxNameLen,length(detectorNames{k}));
end
fprintf(['\n', name,':\n']);
formatString = ['%' sprintf('%d',maxNameLen) 's:'];
fprintf(formatString,'Method name');
for k = 2:size(scores,2)
fprintf('\tImg#%02d',k);
end
fprintf('\n');
for k = 1:numDetectors
fprintf(formatString,detectorNames{k});
for l = 2:size(scores,2)
fprintf('\t%6s',sprintf('%.2f',scores(k,l)));
end
fprintf('\n');
end
end
function plotScores(detectorNames, dataset, score, titleText)
xstart = max([find(sum(score,1) == 0, 1) + 1 1]);
xend = size(score,2);
xLabel = dataset.ImageNamesLabel;
xTicks = dataset.ImageNames;
plot(xstart:xend,score(:,xstart:xend)','+-','linewidth', 2); hold on ;
ylabel(titleText) ;
xlabel(xLabel);
set(gca,'XTick',xstart:1:xend);
set(gca,'XTickLabel',xTicks);
title(titleText);
set(gca,'xtick',1:size(score,2));
maxScore = max([max(max(score)) 1]);
meanEndValue = mean(score(:,xend));
legendLocation = 'SouthEast';
if meanEndValue < maxScore/2
legendLocation = 'NorthEast';
end
legend(detectorNames,'Location',legendLocation);
grid on ;
axis([xstart xend 0 maxScore]);
end
end