more methods and average recall instead of area under the curve

plot_recall_ILSVRC2013.m

@@ -8,42 +8,37 @@ function plot_recall_ILSVRC2013()
 
   val = load('data/ILSVRC2013_val_annotations.mat');
   methods = get_method_configs();
-  methods = methods([3 4 6 8 9 11 12 13 14 15 16]);
+  methods([14 16 19:24]) = [];
 
   valid_methods = compute_best_candidates(val, methods);
   methods = methods(valid_methods);
 
+  plot_legend(methods);
+  printpdf('figures/imagenet_recall_legend.pdf');
+
   fprintf('\n\n');
   for i = 1:numel(methods)
     fprintf('Valid data for plots: %s\n', methods(i).name);
   end
 
   fh = figure;
-  plot_overlap_recall_curve({methods.best_imagenet_candidates_file}, methods, 100, fh, true, 'NorthEast');
+  plot_overlap_recall_curve({methods.best_imagenet_candidates_file}, methods, 100, fh, true, 'none');
   hei = 10;
   wid = 10;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
   set(gcf, 'PaperPositionMode','auto');
   printpdf('figures/imagenet_recall_100.pdf')
 
   fh = figure;
-  plot_overlap_recall_curve({methods.best_imagenet_candidates_file}, methods, 1000, fh, false, 'NorthEast');
+  plot_overlap_recall_curve({methods.best_imagenet_candidates_file}, methods, 1000, fh, false, 'none');
   hei = 10;
   wid = 10;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
   set(gcf, 'PaperPositionMode','auto');
   printpdf('figures/imagenet_recall_1000.pdf')
 
   fh = figure;
-  plot_overlap_recall_curve({methods.best_imagenet_candidates_file}, methods, 1000, fh, true, 'NorthEast');
-  hei = 10;
-  wid = 10;
-  set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
-  set(gcf, 'PaperPositionMode','auto');
-  printpdf('figures/imagenet_recall_1000_with_labels.pdf')
-
-  fh = figure;
-  plot_overlap_recall_curve({methods.best_imagenet_candidates_file}, methods, 10000, fh, false, 'NorthEast');
+  plot_overlap_recall_curve({methods.best_imagenet_candidates_file}, methods, 10000, fh, false, 'none');
   hei = 10;
   wid = 10;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);

plot_recall_voc07.m

@@ -8,34 +8,31 @@ function plot_recall_voc07()
 
   testset = load('data/pascal_voc07_test_annotations.mat');
   methods = get_method_configs();
+  methods([14 16 19:24]) = [];
 
   compute_best_candidates(testset, methods);
+
+  plot_legend(methods);
+  printpdf('figures/recall_legend.pdf');
 
   fh = figure;
-  plot_overlap_recall_curve({methods.best_voc07_candidates_file}, methods, 100, fh, true, 'NorthEast');
+  plot_overlap_recall_curve({methods.best_voc07_candidates_file}, methods, 100, fh, true, 'none');
   hei = 10;
   wid = 10;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
   set(gcf, 'PaperPositionMode','auto');
   printpdf('figures/recall_100.pdf')
 
-  plot_overlap_recall_curve({methods.best_voc07_candidates_file}, methods, 100, fh, true, 'NorthEast', true);
-  hei = 10;
-  wid = 10;
-  set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
-  set(gcf, 'PaperPositionMode','auto');
-  printpdf('figures/recall_100_long_names.pdf')
-
   fh = figure;
-  plot_overlap_recall_curve({methods.best_voc07_candidates_file}, methods, 1000, fh, false, 'NorthEast');
+  plot_overlap_recall_curve({methods.best_voc07_candidates_file}, methods, 1000, fh, false, 'none');
   hei = 10;
   wid = 10;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
   set(gcf, 'PaperPositionMode','auto');
   printpdf('figures/recall_1000.pdf')
 
   fh = figure;
-  plot_overlap_recall_curve({methods.best_voc07_candidates_file}, methods, 10000, fh, false, 'SouthWest');
+  plot_overlap_recall_curve({methods.best_voc07_candidates_file}, methods, 10000, fh, false, 'none');
   hei = 10;
   wid = 10;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);

recall/compute_average_recall.m

@@ -1,4 +1,4 @@
-function [overlap, recall, area] = compute_average_recall(unsorted_overlaps)
+function [overlap, recall, AR] = compute_average_recall(unsorted_overlaps)
   overlap = sort(unsorted_overlaps(:)', 'ascend');
   num_pos = numel(overlap);
   if max(overlap) < 1
@@ -9,9 +9,24 @@
     recall = [1, (num_pos:-1:1)/num_pos];
   end
 
-%   good_overlap = overlap(overlap >= 0.5);
-%   good_recall = recall(overlap >= 0.5);
-  dx = overlap(2:end) - overlap(1:end-1);
-  y = (recall(1:end-1) + recall(2:end)) / 2;
-  area = sum(dx .* y);
+  good_overlap = overlap(overlap >= 0.5);
+  good_recall = recall(overlap >= 0.5);
+  dx = good_overlap(2:end) - good_overlap(1:end-1);
+  y = (good_recall(1:end-1) + good_recall(2:end)) / 2;
+  AR = 2 * sum(dx .* y);
+
+  % resample overlap/recall so the plot points are not too dense
+  delta = 0.005;
+  mask = false(size(overlap));
+  next_overlap = delta;
+  for i = 1:(numel(overlap) - 1)
+    if overlap(i+1) >= next_overlap
+      mask(i) = true;
+      next_overlap = overlap(i) + delta;
+    end
+  end
+  mask(1) = true;
+  mask(end) = true;
+  overlap = overlap(mask);
+  recall = recall(mask);
 end

recall/plot_num_candidates_auc.m

@@ -19,32 +19,32 @@ function plot_num_candidates_auc(iou_files, methods, output_file_prefix)
     y = zeros(num_experiments, 1);
     for exp_idx = 1:num_experiments
       experiment = data.best_candidates(exp_idx);
-      [~, ~, auc] = compute_average_recall(experiment.best_candidates.iou);
+      [~, ~, AR] = compute_average_recall(experiment.best_candidates.iou);
       x(exp_idx) = mean([experiment.image_statistics.num_candidates]);
-      y(exp_idx) = auc;
+      y(exp_idx) = AR;
     end
     line_style = '-';
     if methods(i).is_baseline
       line_style = '--';
     end
+    if ~isempty(methods(i).line_style)
+      line_style = methods(i).line_style;
+    end
     semilogx(x, y, 'Color', methods(i).color, 'LineWidth', 1.5, 'LineStyle', line_style);
     hold on; grid on;
   end
   xlim([10, 10000]);
   ylim([0 1]);
-  xlabel('# candidates'); ylabel('area under recall');
-  legend(labels, 'Location', 'SouthEast');
-  legendshrink(0.5);
-  legend boxoff;
+  xlabel('# proposals'); ylabel('average recall');
   hei = 10;
   wid = 10;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
   set(gcf, 'PaperPositionMode','auto');
-  printpdf(sprintf('figures/%snum_candidates_area_under_recall.pdf', output_file_prefix));
+  printpdf(sprintf('figures/%snum_candidates_average_recall.pdf', output_file_prefix));
 
   % fixed threshold
-  legend_locations = {'SouthEast', 'NorthWest', 'NorthWest'};
-  thresholds = [0.5 0.7 0.8];
+%   legend_locations = {'SouthEast', 'NorthWest', 'NorthWest'};
+  thresholds = [0.5 0.7 0.75 0.8];
   for threshold_i = 1:numel(thresholds)
     threshold = thresholds(threshold_i);
     figure;
@@ -63,20 +63,23 @@ function plot_num_candidates_auc(iou_files, methods, output_file_prefix)
       if methods(i).is_baseline
         line_style = '--';
       end
+      if ~isempty(methods(i).line_style)
+        line_style = methods(i).line_style;
+      end
       semilogx(x, y, 'Color', methods(i).color, 'LineWidth', 1.5, 'LineStyle', line_style);
       hold on; grid on;
     end
     xlim([10, 10000]);
     ylim([0 1]);
-    xlabel('# candidates'); ylabel(sprintf('recall at IoU threshold %.1f', threshold));
-    legend(labels, 'Location', legend_locations{threshold_i});
-    legendshrink(0.5);
-    legend boxoff;
+    xlabel('# proposals'); ylabel(sprintf('recall at IoU threshold %.2f', threshold));
+%     legend(labels, 'Location', legend_locations{threshold_i});
+%     legendshrink(0.5);
+%     legend boxoff;
 %     legend(labels, 'Location', 'SouthEast');
     hei = 10;
     wid = 10;
     set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
     set(gcf, 'PaperPositionMode','auto');
-    printpdf(sprintf('figures/%snum_candidates_recall_%.1f.pdf', output_file_prefix, threshold));
+    printpdf(sprintf('figures/%snum_candidates_recall_%.2f.pdf', output_file_prefix, threshold));
   end
 end

shared/plot_legend.m

@@ -0,0 +1,28 @@
+function plot_legend(method_configs)
+
+  figure; hold on;
+  n_methods = numel(method_configs);
+  [~,order] = sort([method_configs.sort_key]);
+  method_configs = method_configs(order);
+
+  x = 1:10;
+  y = 1:10;
+  handles = zeros([n_methods, 1]);
+  for i = 1:n_methods
+    style = '-';
+    if method_configs(i).is_baseline
+      style = '--';
+    end
+    if ~isempty(method_configs(i).line_style)
+      style = method_configs(i).line_style;
+    end
+    handles(i) = plot(x, y, 'Color', method_configs(i).color, ...
+      'LineWidth', 1.5, 'LineStyle', style);
+  end
+  lh = legend({method_configs.name});
+%   legend boxoff;
+  for i = 1:n_methods
+    set(handles(i), 'visible', 'off');
+  end
+  set(gca, 'visible', 'off');
+end

util/printpdf.m

@@ -3,7 +3,7 @@ function printpdf(fname)
 
 set(gca, 'LooseInset', get(gca, 'TightInset'));
 fname = [regexprep(fname, '^(.*)\.pdf$', '$1'), '.eps'];
-print('-depsc', fname) ;
+print('-depsc', fname);
 if ~system(['epstopdf ', fname])
   system(['rm ', fname]);
 end

util/printpng.m

@@ -0,0 +1,5 @@
+function printpng(fname)
+%PRINTPDF Prints the current figure into a png document
+
+set(gca, 'LooseInset', get(gca, 'TightInset'));
+print('-dpng', '-r200', fname);