Chroma transcription metrics

mir_eval/transcription.py

@@ -335,7 +335,7 @@ def match_note_onsets(ref_intervals, est_intervals, onset_tolerance=0.05,
 
 def match_notes(ref_intervals, ref_pitches, est_intervals, est_pitches,
                 onset_tolerance=0.05, pitch_tolerance=50.0, offset_ratio=0.2,
-                offset_min_tolerance=0.05, strict=False):
+                offset_min_tolerance=0.05, strict=False, chroma=False):
     """Compute a maximum matching between reference and estimated notes,
     subject to onset, pitch and (optionally) offset constraints.
 
@@ -399,6 +399,8 @@ def match_notes(ref_intervals, ref_pitches, est_intervals, est_pitches,
         and pitch matching are performed using ``<=`` (less than or equal). If
         ``strict=True``, the threshold checks are performed using ``<`` (less
         than).
+    chroma : bool
+        Map reference and estimated pitches to chroma (Ignore octave errors)
 
     Returns
     -------
@@ -425,6 +427,10 @@ def match_notes(ref_intervals, ref_pitches, est_intervals, est_pitches,
     # check for pitch matches
     pitch_distances = np.abs(1200*np.subtract.outer(np.log2(ref_pitches),
                                                     np.log2(est_pitches)))
+    if chroma:
+        # 1200 cents is one octave, mod by 1200 to remove octave information
+        pitch_distances = np.mod(pitch_distances, 1200)
+        pitch_distances = np.minimum(1200-pitch_distances,pitch_distances)
     pitch_hit_matrix = cmp_func(pitch_distances, pitch_tolerance)
 
     # check for offset matches if offset_ratio is not None
@@ -467,7 +473,7 @@ def precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals,
                                 est_pitches, onset_tolerance=0.05,
                                 pitch_tolerance=50.0, offset_ratio=0.2,
                                 offset_min_tolerance=0.05, strict=False,
-                                beta=1.0):
+                                beta=1.0,chroma=False):
     """Compute the Precision, Recall and F-measure of correct vs incorrectly
     transcribed notes, and the Average Overlap Ratio for correctly transcribed
     notes (see :func:`average_overlap_ratio`). "Correctness" is determined
@@ -533,6 +539,8 @@ def precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals,
         than).
     beta : float > 0
         Weighting factor for f-measure (default value = 1.0).
+    chroma : bool
+        Map reference and estimated pitches to chroma (Ignore octave errors)
 
     Returns
     -------
@@ -555,7 +563,7 @@ def precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals,
                            pitch_tolerance=pitch_tolerance,
                            offset_ratio=offset_ratio,
                            offset_min_tolerance=offset_min_tolerance,
-                           strict=strict)
+                           strict=strict,chroma=chroma)
 
     precision = float(len(matching))/len(est_pitches)
     recall = float(len(matching))/len(ref_pitches)
@@ -788,7 +796,8 @@ def evaluate(ref_intervals, ref_pitches, est_intervals, est_pitches, **kwargs):
     """
     # Compute all the metrics
     scores = collections.OrderedDict()
-
+
+    kwargs['chroma'] = False
     # Precision, recall and f-measure taking note offsets into account
     kwargs.setdefault('offset_ratio', 0.2)
     orig_offset_ratio = kwargs['offset_ratio']
@@ -809,6 +818,27 @@ def evaluate(ref_intervals, ref_pitches, est_intervals, est_pitches, **kwargs):
         util.filter_kwargs(precision_recall_f1_overlap,
                            ref_intervals, ref_pitches,
                            est_intervals, est_pitches, **kwargs))
+
+    kwargs['chroma'] = True
+    # Precision, recall and f-measure taking note offsets into account
+    kwargs['offset_ratio'] = orig_offset_ratio
+    if kwargs['offset_ratio'] is not None:
+        (scores['Chroma_Precision'],
+         scores['Chroma_Recall'],
+         scores['Chroma_F-measure'],
+         scores['Chroma_Average_Overlap_Ratio']) = util.filter_kwargs(
+            precision_recall_f1_overlap, ref_intervals, ref_pitches,
+            est_intervals, est_pitches, **kwargs)
+
+    # Precision, recall and f-measure NOT taking note offsets into account
+    kwargs['offset_ratio'] = None
+    (scores['Chroma_Precision_no_offset'],
+     scores['Chroma_Recall_no_offset'],
+     scores['Chroma_F-measure_no_offset'],
+     scores['Chroma_Average_Overlap_Ratio_no_offset']) = (
+        util.filter_kwargs(precision_recall_f1_overlap,
+                           ref_intervals, ref_pitches,
+                           est_intervals, est_pitches, **kwargs))
 
     # onset-only metrics
     (scores['Onset_Precision'],

tests/data/transcription/output00.json

@@ -1 +1 @@
-{"Precision": 0.08173076923076923, "Recall": 0.07234042553191489, "F-measure": 0.07674943566591423, "Average_Overlap_Ratio": 0.88768195115369952, "Precision_no_offset": 0.16826923076923078, "Recall_no_offset": 0.14893617021276595, "F-measure_no_offset": 0.1580135440180587, "Average_Overlap_Ratio_no_offset": 0.6158246059728274, "Onset_Precision": 0.30288461538461536, "Onset_Recall": 0.2680851063829787, "Onset_F-measure": 0.28442437923250563, "Offset_Precision": 0.7980769230769231, "Offset_Recall": 0.7063829787234043, "Offset_F-measure": 0.7494356659142212}
+{"Precision": 0.08173076923076923, "Recall": 0.07234042553191489, "F-measure": 0.07674943566591423, "Average_Overlap_Ratio": 0.8876819511536995, "Precision_no_offset": 0.16826923076923078, "Recall_no_offset": 0.14893617021276595, "F-measure_no_offset": 0.1580135440180587, "Average_Overlap_Ratio_no_offset": 0.6158246059728274, "Chroma_Precision": 0.08653846153846154, "Chroma_Recall": 0.07659574468085106, "Chroma_F-measure": 0.08126410835214447, "Chroma_Average_Overlap_Ratio": 0.8954092053280919, "Chroma_Precision_no_offset": 0.20673076923076922, "Chroma_Recall_no_offset": 0.1829787234042553, "Chroma_F-measure_no_offset": 0.19413092550790068, "Chroma_Average_Overlap_Ratio_no_offset": 0.5793980215948555, "Onset_Precision": 0.30288461538461536, "Onset_Recall": 0.2680851063829787, "Onset_F-measure": 0.28442437923250563, "Offset_Precision": 0.7980769230769231, "Offset_Recall": 0.7063829787234043, "Offset_F-measure": 0.7494356659142212}

tests/data/transcription/output01.json

@@ -1 +1 @@
-{"Precision": 0.2446043165467626, "Recall": 0.11486486486486487, "F-measure": 0.15632183908045977, "Average_Overlap_Ratio": 0.76932674055538197, "Precision_no_offset": 0.5467625899280576, "Recall_no_offset": 0.25675675675675674, "F-measure_no_offset": 0.34942528735632183, "Average_Overlap_Ratio_no_offset": 0.58151450278701833, "Onset_Precision": 0.7697841726618705, "Onset_Recall": 0.3614864864864865, "Onset_F-measure": 0.49195402298850577, "Offset_Precision": 0.8345323741007195, "Offset_Recall": 0.3918918918918919, "Offset_F-measure": 0.5333333333333333}
+{"Precision": 0.2446043165467626, "Recall": 0.11486486486486487, "F-measure": 0.15632183908045977, "Average_Overlap_Ratio": 0.769326740555382, "Precision_no_offset": 0.5467625899280576, "Recall_no_offset": 0.25675675675675674, "F-measure_no_offset": 0.34942528735632183, "Average_Overlap_Ratio_no_offset": 0.5815145027870183, "Chroma_Precision": 0.2949640287769784, "Chroma_Recall": 0.13851351351351351, "Chroma_F-measure": 0.18850574712643678, "Chroma_Average_Overlap_Ratio": 0.7832221022721378, "Chroma_Precision_no_offset": 0.5827338129496403, "Chroma_Recall_no_offset": 0.27364864864864863, "Chroma_F-measure_no_offset": 0.3724137931034483, "Chroma_Average_Overlap_Ratio_no_offset": 0.5845481422591593, "Onset_Precision": 0.7697841726618705, "Onset_Recall": 0.3614864864864865, "Onset_F-measure": 0.49195402298850577, "Offset_Precision": 0.8345323741007195, "Offset_Recall": 0.3918918918918919, "Offset_F-measure": 0.5333333333333333}

tests/data/transcription/output02.json

@@ -1 +1 @@
-{"Precision": 0.1164021164021164, "Recall": 0.15942028985507245, "F-measure": 0.1345565749235474, "Average_Overlap_Ratio": 0.871948149157037, "Precision_no_offset": 0.3492063492063492, "Recall_no_offset": 0.4782608695652174, "F-measure_no_offset": 0.40366972477064217, "Average_Overlap_Ratio_no_offset": 0.58729690225319064, "Onset_Precision": 0.4126984126984127, "Onset_Recall": 0.5652173913043478, "Onset_F-measure": 0.4770642201834862, "Offset_Precision": 0.5132275132275133, "Offset_Recall": 0.7028985507246377, "Offset_F-measure": 0.5932721712538227}
+{"Precision": 0.1164021164021164, "Recall": 0.15942028985507245, "F-measure": 0.1345565749235474, "Average_Overlap_Ratio": 0.871948149157037, "Precision_no_offset": 0.3492063492063492, "Recall_no_offset": 0.4782608695652174, "F-measure_no_offset": 0.40366972477064217, "Average_Overlap_Ratio_no_offset": 0.5872969022531906, "Chroma_Precision": 0.1164021164021164, "Chroma_Recall": 0.15942028985507245, "Chroma_F-measure": 0.1345565749235474, "Chroma_Average_Overlap_Ratio": 0.8732221647069539, "Chroma_Precision_no_offset": 0.35978835978835977, "Chroma_Recall_no_offset": 0.4927536231884058, "Chroma_F-measure_no_offset": 0.4159021406727829, "Chroma_Average_Overlap_Ratio_no_offset": 0.5518543030060943, "Onset_Precision": 0.4126984126984127, "Onset_Recall": 0.5652173913043478, "Onset_F-measure": 0.4770642201834862, "Offset_Precision": 0.5132275132275133, "Offset_Recall": 0.7028985507246377, "Offset_F-measure": 0.5932721712538227}

tests/data/transcription/output03.json

@@ -1 +1 @@
-{"Precision": 0.08256880733944955, "Recall": 0.07692307692307693, "F-measure": 0.07964601769911504, "Average_Overlap_Ratio": 0.83144872723186047, "Precision_no_offset": 0.46788990825688076, "Recall_no_offset": 0.4358974358974359, "F-measure_no_offset": 0.45132743362831856, "Average_Overlap_Ratio_no_offset": 0.53625642573827337, "Onset_Precision": 0.5412844036697247, "Onset_Recall": 0.5042735042735043, "Onset_F-measure": 0.5221238938053097, "Offset_Precision": 0.5321100917431193, "Offset_Recall": 0.49572649572649574, "Offset_F-measure": 0.5132743362831859}
+{"Precision": 0.08256880733944955, "Recall": 0.07692307692307693, "F-measure": 0.07964601769911504, "Average_Overlap_Ratio": 0.8314487272318605, "Precision_no_offset": 0.46788990825688076, "Recall_no_offset": 0.4358974358974359, "F-measure_no_offset": 0.45132743362831856, "Average_Overlap_Ratio_no_offset": 0.5362564257382734, "Chroma_Precision": 0.11009174311926606, "Chroma_Recall": 0.10256410256410256, "Chroma_F-measure": 0.10619469026548672, "Chroma_Average_Overlap_Ratio": 0.8086215804589297, "Chroma_Precision_no_offset": 0.47706422018348627, "Chroma_Recall_no_offset": 0.4444444444444444, "Chroma_F-measure_no_offset": 0.46017699115044247, "Chroma_Average_Overlap_Ratio_no_offset": 0.5275356231005376, "Onset_Precision": 0.5412844036697247, "Onset_Recall": 0.5042735042735043, "Onset_F-measure": 0.5221238938053097, "Offset_Precision": 0.5321100917431193, "Offset_Recall": 0.49572649572649574, "Offset_F-measure": 0.5132743362831859}

tests/data/transcription/output04.json

@@ -1 +1 @@
-{"Precision": 0.1270718232044199, "Recall": 0.11917098445595854, "F-measure": 0.12299465240641712, "Average_Overlap_Ratio": 0.79479231600521139, "Precision_no_offset": 0.7734806629834254, "Recall_no_offset": 0.7253886010362695, "F-measure_no_offset": 0.748663101604278, "Average_Overlap_Ratio_no_offset": 0.49770963250522282, "Onset_Precision": 0.861878453038674, "Onset_Recall": 0.8082901554404145, "Onset_F-measure": 0.8342245989304813, "Offset_Precision": 0.6850828729281768, "Offset_Recall": 0.6424870466321243, "Offset_F-measure": 0.6631016042780749}
+{"Precision": 0.1270718232044199, "Recall": 0.11917098445595854, "F-measure": 0.12299465240641712, "Average_Overlap_Ratio": 0.7947923160052114, "Precision_no_offset": 0.7734806629834254, "Recall_no_offset": 0.7253886010362695, "F-measure_no_offset": 0.748663101604278, "Average_Overlap_Ratio_no_offset": 0.4977096325052228, "Chroma_Precision": 0.14917127071823205, "Chroma_Recall": 0.13989637305699482, "Chroma_F-measure": 0.14438502673796794, "Chroma_Average_Overlap_Ratio": 0.7884548843029058, "Chroma_Precision_no_offset": 0.8011049723756906, "Chroma_Recall_no_offset": 0.7512953367875648, "Chroma_F-measure_no_offset": 0.7754010695187165, "Chroma_Average_Overlap_Ratio_no_offset": 0.5031792845177341, "Onset_Precision": 0.861878453038674, "Onset_Recall": 0.8082901554404145, "Onset_F-measure": 0.8342245989304813, "Offset_Precision": 0.6850828729281768, "Offset_Recall": 0.6424870466321243, "Offset_F-measure": 0.6631016042780749}

tests/data/transcription/output05.json

@@ -1 +1 @@
-{"Precision": 0.0, "Recall": 0.0, "F-measure": 0.0, "Average_Overlap_Ratio": 0, "Precision_no_offset": 0.03333333333333333, "Recall_no_offset": 0.07291666666666667, "F-measure_no_offset": 0.0457516339869281, "Average_Overlap_Ratio_no_offset": 0.41481713300167772, "Onset_Precision": 0.1523809523809524, "Onset_Recall": 0.3333333333333333, "Onset_F-measure": 0.20915032679738563, "Offset_Precision": 0.38571428571428573, "Offset_Recall": 0.84375, "Offset_F-measure": 0.5294117647058824}
+{"Precision": 0.0, "Recall": 0.0, "F-measure": 0.0, "Average_Overlap_Ratio": 0, "Precision_no_offset": 0.03333333333333333, "Recall_no_offset": 0.07291666666666667, "F-measure_no_offset": 0.0457516339869281, "Average_Overlap_Ratio_no_offset": 0.4148171330016777, "Chroma_Precision": 0.0, "Chroma_Recall": 0.0, "Chroma_F-measure": 0.0, "Chroma_Average_Overlap_Ratio": 0, "Chroma_Precision_no_offset": 0.05238095238095238, "Chroma_Recall_no_offset": 0.11458333333333333, "Chroma_F-measure_no_offset": 0.0718954248366013, "Chroma_Average_Overlap_Ratio_no_offset": 0.3026633566011679, "Onset_Precision": 0.1523809523809524, "Onset_Recall": 0.3333333333333333, "Onset_F-measure": 0.20915032679738563, "Offset_Precision": 0.38571428571428573, "Offset_Recall": 0.84375, "Offset_F-measure": 0.5294117647058824}

tests/data/transcription/output06.json

@@ -1 +1 @@
-{"Precision": 0.2, "Recall": 0.18181818181818182, "F-measure": 0.1904761904761905, "Average_Overlap_Ratio": 0.73602077166935231, "Precision_no_offset": 0.37857142857142856, "Recall_no_offset": 0.34415584415584416, "F-measure_no_offset": 0.36054421768707484, "Average_Overlap_Ratio_no_offset": 0.60104325916355583, "Onset_Precision": 0.6928571428571428, "Onset_Recall": 0.6298701298701299, "Onset_F-measure": 0.6598639455782312, "Offset_Precision": 0.7357142857142858, "Offset_Recall": 0.6688311688311688, "Offset_F-measure": 0.7006802721088435}
+{"Precision": 0.2, "Recall": 0.18181818181818182, "F-measure": 0.1904761904761905, "Average_Overlap_Ratio": 0.7360207716693523, "Precision_no_offset": 0.37857142857142856, "Recall_no_offset": 0.34415584415584416, "F-measure_no_offset": 0.36054421768707484, "Average_Overlap_Ratio_no_offset": 0.6010432591635558, "Chroma_Precision": 0.2357142857142857, "Chroma_Recall": 0.21428571428571427, "Chroma_F-measure": 0.22448979591836735, "Chroma_Average_Overlap_Ratio": 0.7322035953385762, "Chroma_Precision_no_offset": 0.4642857142857143, "Chroma_Recall_no_offset": 0.42207792207792205, "Chroma_F-measure_no_offset": 0.4421768707482993, "Chroma_Average_Overlap_Ratio_no_offset": 0.5726489871212412, "Onset_Precision": 0.6928571428571428, "Onset_Recall": 0.6298701298701299, "Onset_F-measure": 0.6598639455782312, "Offset_Precision": 0.7357142857142858, "Offset_Recall": 0.6688311688311688, "Offset_F-measure": 0.7006802721088435}

tests/data/transcription/output07.json

@@ -1 +1 @@
-{"Precision": 0.13227513227513227, "Recall": 0.07861635220125786, "F-measure": 0.09861932938856015, "Average_Overlap_Ratio": 0.73459052448613871, "Precision_no_offset": 0.4074074074074074, "Recall_no_offset": 0.24213836477987422, "F-measure_no_offset": 0.3037475345167653, "Average_Overlap_Ratio_no_offset": 0.50249110561442467, "Onset_Precision": 0.708994708994709, "Onset_Recall": 0.42138364779874216, "Onset_F-measure": 0.5285996055226824, "Offset_Precision": 0.7037037037037037, "Offset_Recall": 0.41823899371069184, "Offset_F-measure": 0.52465483234714}
+{"Precision": 0.13227513227513227, "Recall": 0.07861635220125786, "F-measure": 0.09861932938856015, "Average_Overlap_Ratio": 0.7345905244861387, "Precision_no_offset": 0.4074074074074074, "Recall_no_offset": 0.24213836477987422, "F-measure_no_offset": 0.3037475345167653, "Average_Overlap_Ratio_no_offset": 0.5024911056144247, "Chroma_Precision": 0.15873015873015872, "Chroma_Recall": 0.09433962264150944, "Chroma_F-measure": 0.11834319526627221, "Chroma_Average_Overlap_Ratio": 0.7374347313654074, "Chroma_Precision_no_offset": 0.5026455026455027, "Chroma_Recall_no_offset": 0.29874213836477986, "Chroma_F-measure_no_offset": 0.3747534516765286, "Chroma_Average_Overlap_Ratio_no_offset": 0.4423976845729225, "Onset_Precision": 0.708994708994709, "Onset_Recall": 0.42138364779874216, "Onset_F-measure": 0.5285996055226824, "Offset_Precision": 0.7037037037037037, "Offset_Recall": 0.41823899371069184, "Offset_F-measure": 0.52465483234714}

tests/data/transcription/output08.json

@@ -1 +1 @@
-{"Precision": 0.19270833333333334, "Recall": 0.09226932668329177, "F-measure": 0.12478920741989884, "Average_Overlap_Ratio": 0.72256747506016228, "Precision_no_offset": 0.453125, "Recall_no_offset": 0.2169576059850374, "F-measure_no_offset": 0.2934232715008432, "Average_Overlap_Ratio_no_offset": 0.48476467726821298, "Onset_Precision": 0.6458333333333334, "Onset_Recall": 0.3092269326683292, "Onset_F-measure": 0.41821247892074204, "Offset_Precision": 0.6875, "Offset_Recall": 0.32917705735660846, "Offset_F-measure": 0.44519392917369305}
+{"Precision": 0.19270833333333334, "Recall": 0.09226932668329177, "F-measure": 0.12478920741989884, "Average_Overlap_Ratio": 0.7225674750601623, "Precision_no_offset": 0.453125, "Recall_no_offset": 0.2169576059850374, "F-measure_no_offset": 0.2934232715008432, "Average_Overlap_Ratio_no_offset": 0.484764677268213, "Chroma_Precision": 0.28125, "Chroma_Recall": 0.13466334164588528, "Chroma_F-measure": 0.1821247892074199, "Chroma_Average_Overlap_Ratio": 0.6909010850532998, "Chroma_Precision_no_offset": 0.6041666666666666, "Chroma_Recall_no_offset": 0.2892768079800499, "Chroma_F-measure_no_offset": 0.3912310286677909, "Chroma_Average_Overlap_Ratio_no_offset": 0.4934403443807876, "Onset_Precision": 0.6458333333333334, "Onset_Recall": 0.3092269326683292, "Onset_F-measure": 0.41821247892074204, "Offset_Precision": 0.6875, "Offset_Recall": 0.32917705735660846, "Offset_F-measure": 0.44519392917369305}

tests/data/transcription/output09.json

@@ -1 +1 @@
-{"Precision": 0.13191489361702127, "Recall": 0.09967845659163987, "F-measure": 0.11355311355311355, "Average_Overlap_Ratio": 0.81129809533273478, "Precision_no_offset": 0.42127659574468085, "Recall_no_offset": 0.3183279742765273, "F-measure_no_offset": 0.3626373626373626, "Average_Overlap_Ratio_no_offset": 0.56517655829605251, "Onset_Precision": 0.6170212765957447, "Onset_Recall": 0.4662379421221865, "Onset_F-measure": 0.531135531135531, "Offset_Precision": 0.7574468085106383, "Offset_Recall": 0.572347266881029, "Offset_F-measure": 0.652014652014652}
+{"Precision": 0.13191489361702127, "Recall": 0.09967845659163987, "F-measure": 0.11355311355311355, "Average_Overlap_Ratio": 0.8112980953327348, "Precision_no_offset": 0.42127659574468085, "Recall_no_offset": 0.3183279742765273, "F-measure_no_offset": 0.3626373626373626, "Average_Overlap_Ratio_no_offset": 0.5651765582960525, "Chroma_Precision": 0.1702127659574468, "Chroma_Recall": 0.12861736334405144, "Chroma_F-measure": 0.14652014652014653, "Chroma_Average_Overlap_Ratio": 0.8096273569645686, "Chroma_Precision_no_offset": 0.4851063829787234, "Chroma_Recall_no_offset": 0.3665594855305466, "Chroma_F-measure_no_offset": 0.4175824175824176, "Chroma_Average_Overlap_Ratio_no_offset": 0.5315499758243736, "Onset_Precision": 0.6170212765957447, "Onset_Recall": 0.4662379421221865, "Onset_F-measure": 0.531135531135531, "Offset_Precision": 0.7574468085106383, "Offset_Recall": 0.572347266881029, "Offset_F-measure": 0.652014652014652}