Allow classification results to deviate slightly

crates/openvino/tests/classify-alexnet.rs

@@ -1,11 +1,12 @@
 //! Demonstrates using `openvino-rs` to classify an image using an AlexNet model and a prepared input tensor. See
 //! [README](fixtures/alexnet/README.md) for details on how this test fixture was prepared.
 mod fixtures;
+mod util;
 
 use fixtures::alexnet::Fixture;
-use float_cmp::approx_eq;
 use openvino::{Blob, Core, Layout, Precision, TensorDesc};
 use std::fs;
+use util::{Prediction, Predictions};
 
 #[test]
 fn classify_alexnet() {
@@ -38,20 +39,20 @@ fn classify_alexnet() {
     let buffer = unsafe { results.buffer_mut_as_type::<f32>().unwrap().to_vec() };
 
     // Sort results.
-    let mut results: Results = buffer
+    let mut results: Predictions = buffer
         .iter()
         .enumerate()
-        .map(|(c, p)| Result(c, *p))
+        .map(|(c, p)| Prediction::new(c, *p))
         .collect();
-    results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
+    results.sort();
 
-    // Compare results using approximate FP comparisons; annotated with classification tag from
+    // Compare results using approximate FP comparisons; annotated with classification tags from
     // https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a.
-    results[0].assert_approx_eq(&Result(963, 0.5321184)); // pizza
-    results[1].assert_approx_eq(&Result(923, 0.1050855)); // plate
-    results[2].assert_approx_eq(&Result(926, 0.1022315)); // hot pot
-    results[3].assert_approx_eq(&Result(909, 0.0614674)); // wok
-    results[4].assert_approx_eq(&Result(762, 0.0549604)); // restaurant
+    results[0].assert_approx_eq((963, 0.5321184)); // pizza
+    results[1].assert_approx_eq((923, 0.1050855)); // plate
+    results[2].assert_approx_eq((926, 0.1022315)); // hot pot
+    results[3].assert_approx_eq((909, 0.0614674)); // wok
+    results[4].assert_approx_eq((762, 0.0549604)); // restaurant
 
     // This above results match the output of running OpenVINO's `hello_classification` with the same inputs:
     // $ bin/intel64/Debug/hello_classification /tmp/alexnet/bvlc_alexnet.xml /tmp/alexnet/val2017/000000062808.jpg CPU
@@ -70,25 +71,3 @@ fn classify_alexnet() {
     // 935     0.0130160
     // 965     0.0094148
 }
-
-/// A structure for holding the `(category, probability)` pair extracted from the output tensor of
-/// the OpenVINO classification.
-#[derive(Debug, PartialEq)]
-struct Result(usize, f32);
-type Results = Vec<Result>;
-
-impl Result {
-    fn assert_approx_eq(&self, expected: &Result) {
-        assert_eq!(
-            self.0, expected.0,
-            "Expected class ID {} but found {}",
-            expected.0, self.0
-        );
-        let approx_matches = approx_eq!(f32, self.1, expected.1, ulps = 2, epsilon = 0.01);
-        assert!(
-            approx_matches,
-            "Expected probability {} but found {} (outside of tolerance)",
-            expected.1, self.1
-        );
-    }
-}

crates/openvino/tests/classify-inception.rs

@@ -1,10 +1,12 @@
 //! Demonstrates using `openvino-rs` to classify an image using an Inception SSD model and a prepared input tensor. See
 //! [README](fixtures/inception/README.md) for details on how this test fixture was prepared.
 mod fixtures;
+mod util;
 
 use fixtures::inception::Fixture;
-use openvino::{Blob, Core, Layout, Precision, ResizeAlgorithm, TensorDesc};
+use openvino::{Blob, Core, Layout, Precision, TensorDesc};
 use std::fs;
+use util::{Prediction, Predictions};
 
 #[test]
 fn classify_inception() {
@@ -37,23 +39,19 @@ fn classify_inception() {
     let buffer = unsafe { results.buffer_mut_as_type::<f32>().unwrap().to_vec() };
 
     // Sort results.
-    let mut results: Results = buffer
+    let mut results: Predictions = buffer
         .iter()
         .enumerate()
-        .map(|(c, p)| Result(c, *p))
+        .map(|(c, p)| Prediction::new(c, *p))
         .collect();
-    results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
+    results.sort();
 
-    assert_eq!(
-        &results[..5],
-        &[
-            Result(964, 0.9648312),
-            Result(763, 0.0015633557),
-            Result(412, 0.0007776478),
-            Result(814, 0.0006391522),
-            Result(924, 0.0006150733),
-        ][..]
-    )
+    // Note that these results appear to be off-by-one: pizza should be ID 963.
+    results[0].assert_approx_eq((964, 0.9648312));
+    results[1].assert_approx_eq((763, 0.0015633557));
+    results[2].assert_approx_eq((412, 0.0007776478));
+    results[3].assert_approx_eq((814, 0.0006391522));
+    results[4].assert_approx_eq((924, 0.0006150733));
 
     // The results above almost match the output of OpenVINO's `hello_classification` with similar
     // inputs:
@@ -73,9 +71,3 @@ fn classify_inception() {
     // 927     0.0003644
     // 923     0.0002908
 }
-
-/// A structure for holding the `(category, probability)` pair extracted from the output tensor of
-/// the OpenVINO classification.
-#[derive(Debug, PartialEq)]
-struct Result(usize, f32);
-type Results = Vec<Result>;

crates/openvino/tests/classify-mobilenet.rs

@@ -2,11 +2,12 @@
 //! input tensor. See [README](fixtures/inception/README.md) for details on how this test fixture
 //! was prepared.
 mod fixtures;
+mod util;
 
 use fixtures::mobilenet::Fixture;
-use float_cmp::approx_eq;
 use openvino::{Blob, Core, Layout, Precision, TensorDesc};
 use std::fs;
+use util::{Prediction, Predictions};
 
 #[test]
 fn classify_mobilenet() {
@@ -41,21 +42,21 @@ fn classify_mobilenet() {
     // Sort results. It is unclear why the MobileNet output indices are "off by one" but the
     // `.skip(1)` below seems necessary to get results that make sense (e.g. 763 = "revolver" vs 762
     // = "restaurant").
-    let mut results: Results = buffer
+    let mut results: Predictions = buffer
         .iter()
         .skip(1)
         .enumerate()
-        .map(|(c, p)| Result(c, *p))
+        .map(|(c, p)| Prediction::new(c, *p))
         .collect();
-    results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
+    results.sort();
 
-    // Compare results using approximate FP comparisons; annotated with classification tag from
+    // Compare results using approximate FP comparisons; annotated with classification tags from
     // https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a.
-    results[0].assert_approx_eq(&Result(963, 0.7134405)); // pizza
-    results[1].assert_approx_eq(&Result(762, 0.0715866)); // restaurant
-    results[2].assert_approx_eq(&Result(909, 0.0360171)); // wok
-    results[3].assert_approx_eq(&Result(926, 0.0160412)); // hot pot
-    results[4].assert_approx_eq(&Result(567, 0.0152781)); // frying pan
+    results[0].assert_approx_eq((963, 0.7134405)); // pizza
+    results[1].assert_approx_eq((762, 0.0715866)); // restaurant
+    results[2].assert_approx_eq((909, 0.0360171)); // wok
+    results[3].assert_approx_eq((926, 0.0160412)); // hot pot
+    results[4].assert_approx_eq((567, 0.0152781)); // frying pan
 
     // This above results almost match (see "off by one" comment above) the output of running
     // OpenVINO's `hello_classification` with the same inputs:
@@ -74,25 +75,3 @@ fn classify_mobilenet() {
     // 965     0.0058377
     // 545     0.0043731
 }
-
-/// A structure for holding the `(category, probability)` pair extracted from the output tensor of
-/// the OpenVINO classification.
-#[derive(Debug, PartialEq)]
-struct Result(usize, f32);
-type Results = Vec<Result>;
-
-impl Result {
-    fn assert_approx_eq(&self, expected: &Result) {
-        assert_eq!(
-            self.0, expected.0,
-            "Expected class ID {} but found {}",
-            expected.0, self.0
-        );
-        let approx_matches = approx_eq!(f32, self.1, expected.1, ulps = 2, epsilon = 0.01);
-        assert!(
-            approx_matches,
-            "Expected probability {} but found {} (outside of tolerance)",
-            expected.1, self.1
-        );
-    }
-}

crates/openvino/tests/util.rs

@@ -0,0 +1,81 @@
+use core::cmp::Ordering;
+use float_cmp::{ApproxEq, F32Margin};
+
+/// A structure for holding the `(category, probability)` pair extracted from the output tensor of
+/// the OpenVINO classification.
+#[derive(Debug)]
+pub struct Prediction {
+    id: usize,
+    prob: f32,
+}
+
+impl Prediction {
+    pub fn new(id: usize, prob: f32) -> Self {
+        Self { id, prob }
+    }
+
+    /// Reduce the boilerplate to assert that two predictions are approximately the same.
+    pub fn assert_approx_eq<P: Into<Self>>(&self, expected: P) {
+        let expected = expected.into();
+        assert_eq!(
+            self.id, expected.id,
+            "Expected class ID {} but found {}",
+            expected.id, self.id
+        );
+        let approx_matches = self.approx_eq(&expected, DEFAULT_MARGIN);
+        assert!(
+            approx_matches,
+            "Expected probability {} but found {} (outside of default margin of error)",
+            expected.prob, self.prob
+        );
+    }
+}
+
+impl From<(usize, f32)> for Prediction {
+    fn from(p: (usize, f32)) -> Self {
+        Prediction::new(p.0, p.1)
+    }
+}
+
+/// Classification results are ordered by their probability, from greatest to smallest.
+impl Ord for Prediction {
+    fn cmp(&self, other: &Self) -> Ordering {
+        assert!(!self.prob.is_nan());
+        assert!(!other.prob.is_nan());
+        other
+            .prob
+            .partial_cmp(&self.prob)
+            .expect("a comparable value")
+    }
+}
+
+impl PartialOrd for Prediction {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for Prediction {
+    fn eq(&self, other: &Self) -> bool {
+        self.prob == other.prob
+    }
+}
+
+impl Eq for Prediction {}
+
+impl ApproxEq for &Prediction {
+    type Margin = F32Margin;
+    fn approx_eq<T: Into<Self::Margin>>(self, other: Self, margin: T) -> bool {
+        let margin = margin.into();
+        self.prob.approx_eq(other.prob, margin)
+    }
+}
+
+/// The default margin for error allowed for comparing classification results.
+pub const DEFAULT_MARGIN: F32Margin = F32Margin {
+    epsilon: 0.01,
+    ulps: 2,
+};
+
+/// A helper type for manipulating lists of results.
+pub type Predictions = Vec<Prediction>;