Permalink
Browse files

Improve precision-recall script, add results

  • Loading branch information...
1 parent 1df989d commit 0ccab9259b40ea910f6fc97db98b7501142e53a8 @weltenwort committed Oct 4, 2012
@@ -59,12 +59,14 @@ def execute(self, args, config, study):
#self.logger.log(signature)
data["distances"][image_set["query_image"]][source_image_filename] =\
common.load(config["metric"]["metric"]).execute(query_signature, signature, data=data)
- #self.logger.log("Distance: {}".format(a))
+ self.logger.log("Calculating precisions for '{}'...".format(image_set["query_image"]))
+ a = data["precisions"][image_set["query_image"]] = self.get_precision_recall(image_set["query_image"], data["distances"][image_set["query_image"]], study)
+ self.logger.log("Precisions: {}".format(a))
#correlations, mean_correlation = self.correlate_to_study(data["distances"], study)
- precision_recall_stats, mean_stats = self.correlate_to_study(data["distances"], study)
+ #precision_recall_stats, mean_stats = self.correlate_to_study(data["distances"], study)
#self.logger.log("Mean correlation: {}".format(mean_correlation))
- return (precision_recall_stats, mean_stats)
+ return (data["precisions"], self.get_mean_average_precision(data["precisions"]))
if __name__ == "__main__":
CodebookFeaturesBenchmark()()
@@ -289,3 +289,85 @@ def correlate_to_study(self, distances, study):
image_set["distances"])[0] for query_image, image_set\
in benchmark_data.iteritems()}
return results, numpy.mean(results.values())
+
+
+@ApplicationBase.argument("-s", "--study", action="store", dest="study",\
+ required=True)
+@ApplicationBase.argument("-o", "--output-filename", action="store", dest="output_filename",\
+ default=None)
+class PRBenchmarkBase(ApplicationBase):
+ DEFAULT_COMMAND = "execute"
+
+ def load_study(self, args):
+ return self.load_json(args.study)
+
+ def dispatch_subcommand(self, args, config, **kwargs):
+ config_path = pathlib.Path(args.config)
+ output_filename = args.output_filename or "r_{base}.{timestamp}.json".format(
+ base=config_path.parts[-1][:-len(config_path.ext)],
+ timestamp=datetime.datetime.now().isoformat(),
+ )
+ study = self.load_study(args)
+ old_stdout = sys.stdout
+ sys.stdout = open(os.devnull, 'w')
+ try:
+ start_time = datetime.datetime.now()
+ correlations, mean_correlation = super(BenchmarkBase, self)\
+ .dispatch_subcommand(args, config, study=study)
+ stop_time = datetime.datetime.now()
+ finally:
+ sys.stdout = old_stdout
+ duration = (stop_time - start_time).total_seconds()
+ result_summary = json.dumps(dict(
+ correlations=correlations,
+ mean_correlation=mean_correlation,
+ datetime=str(start_time),
+ duration=duration,
+ config=config,
+ ), indent=4)
+ with open(output_filename, "w") as fp:
+ fp.write(result_summary)
+ self.logger.log("Duration: {}\nMean Correlation: {}".format(duration, mean_correlation))
+
+ def get_benchmark_for_distances(self, distances, study):
+ benchmark = {}
+ for query_image, images in distances.iteritems():
+ benchmark_row = []
+ distance_row = []
+ for image, distance in images.iteritems():
+ benchmark_row.append(study[query_image][image])
+ distance_row.append(distance)
+ benchmark[query_image] = dict(
+ benchmark=benchmark_row,
+ distances=distance_row,
+ )
+ return benchmark
+
+ def get_precision_recall(self, query_image_filename, category_distances, study):
+ results = []
+ recall_indices = [int(i) for i in numpy.linspace(1, 0, 10, endpoint=False)]
+ recall_breakpoint = recall_indices.pop(-1)
+
+ max_count = len(study[query_image_filename])
+ count = 0
+ positive_count = 0
+ for source_image_filename, distance in sorted(category_distances.items(), key=lambda x: x[1]):
+ count += 1
+ if source_image_filename in study[query_image_filename]:
+ positive_count += 1
+ recall = float(positive_count) / max_count
+ if recall >= recall_breakpoint:
+ recall_breakpoint = recall_indices.pop(-1)
+ precision = positive_count / count
+ results.append((recall, precision))
+ return results
+
+ def get_mean_average_precision(self, pr_map):
+ return numpy.mean([numpy.mean(v, axis=0)[0] for v in pr_map.values()])
+
+ def correlate_to_study(self, all_distances, study):
+ results = {}
+ for query_image_filename, category_distances in all_distances:
+ self.logger.log("Evaluating query")
+ #results[query_image_filename] =
+ return results, numpy.mean(results.values())
@@ -0,0 +1,252 @@
+{
+ "images": [
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/0.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/0/*.jpg"
+ ],
+ "key": "set0"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/1.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/1/*.jpg"
+ ],
+ "key": "set1"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/2.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/2/*.jpg"
+ ],
+ "key": "set2"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/4.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/4/*.jpg"
+ ],
+ "key": "set4"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/6.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/6/*.jpg"
+ ],
+ "key": "set6"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/7.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/7/*.jpg"
+ ],
+ "key": "set7"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/8.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/8/*.jpg"
+ ],
+ "key": "set8"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/9.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/9/*.jpg"
+ ],
+ "key": "set9"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/10.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/10/*.jpg"
+ ],
+ "key": "set10"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/11.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/11/*.jpg"
+ ],
+ "key": "set11"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/12.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/12/*.jpg"
+ ],
+ "key": "set12"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/13.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/13/*.jpg"
+ ],
+ "key": "set13"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/14.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/14/*.jpg"
+ ],
+ "key": "set14"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/20.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/20/*.jpg"
+ ],
+ "key": "set20"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/21.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/21/*.jpg"
+ ],
+ "key": "set21"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/23.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/23/*.jpg"
+ ],
+ "key": "set23"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/24.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/24/*.jpg"
+ ],
+ "key": "set24"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/25.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/25/*.jpg"
+ ],
+ "key": "set25"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/26.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/26/*.jpg"
+ ],
+ "key": "set26"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/27.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/27/*.jpg"
+ ],
+ "key": "set27"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/28.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/28/*.jpg"
+ ],
+ "key": "set28"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/30.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/30/*.jpg"
+ ],
+ "key": "set30"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/33.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/33/*.jpg"
+ ],
+ "key": "set33"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/34.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/34/*.jpg"
+ ],
+ "key": "set34"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/38.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/38/*.jpg"
+ ],
+ "key": "set38"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/42.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/42/*.jpg"
+ ],
+ "key": "set42"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/43.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/43/*.jpg"
+ ],
+ "key": "set43"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/45.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/45/*.jpg"
+ ],
+ "key": "set45"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/46.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/46/*.jpg"
+ ],
+ "key": "set46"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/47.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/47/*.jpg"
+ ],
+ "key": "set47"
+ },
+ {
+ "query_image": "/home/laeroth/Downloads/benchmark/sketches/48.png",
+ "source_images": [
+ "/home/laeroth/Downloads/benchmark/images/48/*.jpg"
+ ],
+ "key": "set48"
+ }
+ ],
+ "curvelets": {
+ "transform": "transforms.curvelet_strictangles",
+ "angles": 12,
+ "scales": 4
+ },
+ "readers": {
+ "query": "readers.luma",
+ "image": "readers.canny",
+ "canny_sigma": 2.5
+ },
+ "features": {
+ "extractor": "features.patch_means",
+ "grid_size": 8,
+ "patch_size": 3
+ },
+ "codebook": {
+ "codebook_size": 1000,
+ "metric": "kmeans"
+ },
+ "weights": {
+ "use_weights": "weights.tfidf",
+ "use_stopwords": false
+ },
+ "metric": {
+ "metric": "metrics.histogram_intersection"
+ },
+ "cache": {
+ "feature_enabled": true,
+ "reader_enabled": true,
+ "reader_path": "/media/data0/unsorted/caches"
+ }
+}
Oops, something went wrong.

0 comments on commit 0ccab92

Please sign in to comment.