From 08ebb0be91a766fc824363a85db1446222c2cae5 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Fri, 26 Mar 2021 15:40:27 -0700 Subject: [PATCH] Allow classification results to deviate slightly This change introduces a test utility, `util::Prediction`, that simplifies the classification tests. Using the `float-cmp` crate, it checks that the expected results have the correct ID and that the classification probability matches within a certain `DEFAULT_MARGIN` of error. --- crates/openvino/tests/classify-alexnet.rs | 43 +++-------- crates/openvino/tests/classify-inception.rs | 32 +++----- crates/openvino/tests/classify-mobilenet.rs | 43 +++-------- crates/openvino/tests/util.rs | 81 +++++++++++++++++++++ 4 files changed, 115 insertions(+), 84 deletions(-) create mode 100644 crates/openvino/tests/util.rs diff --git a/crates/openvino/tests/classify-alexnet.rs b/crates/openvino/tests/classify-alexnet.rs index e3c5633..b4be39e 100644 --- a/crates/openvino/tests/classify-alexnet.rs +++ b/crates/openvino/tests/classify-alexnet.rs @@ -1,11 +1,12 @@ //! Demonstrates using `openvino-rs` to classify an image using an AlexNet model and a prepared input tensor. See //! [README](fixtures/alexnet/README.md) for details on how this test fixture was prepared. mod fixtures; +mod util; use fixtures::alexnet::Fixture; -use float_cmp::approx_eq; use openvino::{Blob, Core, Layout, Precision, TensorDesc}; use std::fs; +use util::{Prediction, Predictions}; #[test] fn classify_alexnet() { @@ -38,20 +39,20 @@ fn classify_alexnet() { let buffer = unsafe { results.buffer_mut_as_type::().unwrap().to_vec() }; // Sort results. - let mut results: Results = buffer + let mut results: Predictions = buffer .iter() .enumerate() - .map(|(c, p)| Result(c, *p)) + .map(|(c, p)| Prediction::new(c, *p)) .collect(); - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); + results.sort(); - // Compare results using approximate FP comparisons; annotated with classification tag from + // Compare results using approximate FP comparisons; annotated with classification tags from // https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a. - results[0].assert_approx_eq(&Result(963, 0.5321184)); // pizza - results[1].assert_approx_eq(&Result(923, 0.1050855)); // plate - results[2].assert_approx_eq(&Result(926, 0.1022315)); // hot pot - results[3].assert_approx_eq(&Result(909, 0.0614674)); // wok - results[4].assert_approx_eq(&Result(762, 0.0549604)); // restaurant + results[0].assert_approx_eq((963, 0.5321184)); // pizza + results[1].assert_approx_eq((923, 0.1050855)); // plate + results[2].assert_approx_eq((926, 0.1022315)); // hot pot + results[3].assert_approx_eq((909, 0.0614674)); // wok + results[4].assert_approx_eq((762, 0.0549604)); // restaurant // This above results match the output of running OpenVINO's `hello_classification` with the same inputs: // $ bin/intel64/Debug/hello_classification /tmp/alexnet/bvlc_alexnet.xml /tmp/alexnet/val2017/000000062808.jpg CPU @@ -70,25 +71,3 @@ fn classify_alexnet() { // 935 0.0130160 // 965 0.0094148 } - -/// A structure for holding the `(category, probability)` pair extracted from the output tensor of -/// the OpenVINO classification. -#[derive(Debug, PartialEq)] -struct Result(usize, f32); -type Results = Vec; - -impl Result { - fn assert_approx_eq(&self, expected: &Result) { - assert_eq!( - self.0, expected.0, - "Expected class ID {} but found {}", - expected.0, self.0 - ); - let approx_matches = approx_eq!(f32, self.1, expected.1, ulps = 2, epsilon = 0.01); - assert!( - approx_matches, - "Expected probability {} but found {} (outside of tolerance)", - expected.1, self.1 - ); - } -} diff --git a/crates/openvino/tests/classify-inception.rs b/crates/openvino/tests/classify-inception.rs index 784cd2d..fbafb51 100644 --- a/crates/openvino/tests/classify-inception.rs +++ b/crates/openvino/tests/classify-inception.rs @@ -1,10 +1,12 @@ //! Demonstrates using `openvino-rs` to classify an image using an Inception SSD model and a prepared input tensor. See //! [README](fixtures/inception/README.md) for details on how this test fixture was prepared. mod fixtures; +mod util; use fixtures::inception::Fixture; -use openvino::{Blob, Core, Layout, Precision, ResizeAlgorithm, TensorDesc}; +use openvino::{Blob, Core, Layout, Precision, TensorDesc}; use std::fs; +use util::{Prediction, Predictions}; #[test] fn classify_inception() { @@ -37,23 +39,19 @@ fn classify_inception() { let buffer = unsafe { results.buffer_mut_as_type::().unwrap().to_vec() }; // Sort results. - let mut results: Results = buffer + let mut results: Predictions = buffer .iter() .enumerate() - .map(|(c, p)| Result(c, *p)) + .map(|(c, p)| Prediction::new(c, *p)) .collect(); - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); + results.sort(); - assert_eq!( - &results[..5], - &[ - Result(964, 0.9648312), - Result(763, 0.0015633557), - Result(412, 0.0007776478), - Result(814, 0.0006391522), - Result(924, 0.0006150733), - ][..] - ) + // Note that these results appear to be off-by-one: pizza should be ID 963. + results[0].assert_approx_eq((964, 0.9648312)); + results[1].assert_approx_eq((763, 0.0015633557)); + results[2].assert_approx_eq((412, 0.0007776478)); + results[3].assert_approx_eq((814, 0.0006391522)); + results[4].assert_approx_eq((924, 0.0006150733)); // The results above almost match the output of OpenVINO's `hello_classification` with similar // inputs: @@ -73,9 +71,3 @@ fn classify_inception() { // 927 0.0003644 // 923 0.0002908 } - -/// A structure for holding the `(category, probability)` pair extracted from the output tensor of -/// the OpenVINO classification. -#[derive(Debug, PartialEq)] -struct Result(usize, f32); -type Results = Vec; diff --git a/crates/openvino/tests/classify-mobilenet.rs b/crates/openvino/tests/classify-mobilenet.rs index 5ab5cf3..8a8d8ec 100644 --- a/crates/openvino/tests/classify-mobilenet.rs +++ b/crates/openvino/tests/classify-mobilenet.rs @@ -2,11 +2,12 @@ //! input tensor. See [README](fixtures/inception/README.md) for details on how this test fixture //! was prepared. mod fixtures; +mod util; use fixtures::mobilenet::Fixture; -use float_cmp::approx_eq; use openvino::{Blob, Core, Layout, Precision, TensorDesc}; use std::fs; +use util::{Prediction, Predictions}; #[test] fn classify_mobilenet() { @@ -41,21 +42,21 @@ fn classify_mobilenet() { // Sort results. It is unclear why the MobileNet output indices are "off by one" but the // `.skip(1)` below seems necessary to get results that make sense (e.g. 763 = "revolver" vs 762 // = "restaurant"). - let mut results: Results = buffer + let mut results: Predictions = buffer .iter() .skip(1) .enumerate() - .map(|(c, p)| Result(c, *p)) + .map(|(c, p)| Prediction::new(c, *p)) .collect(); - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); + results.sort(); - // Compare results using approximate FP comparisons; annotated with classification tag from + // Compare results using approximate FP comparisons; annotated with classification tags from // https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a. - results[0].assert_approx_eq(&Result(963, 0.7134405)); // pizza - results[1].assert_approx_eq(&Result(762, 0.0715866)); // restaurant - results[2].assert_approx_eq(&Result(909, 0.0360171)); // wok - results[3].assert_approx_eq(&Result(926, 0.0160412)); // hot pot - results[4].assert_approx_eq(&Result(567, 0.0152781)); // frying pan + results[0].assert_approx_eq((963, 0.7134405)); // pizza + results[1].assert_approx_eq((762, 0.0715866)); // restaurant + results[2].assert_approx_eq((909, 0.0360171)); // wok + results[3].assert_approx_eq((926, 0.0160412)); // hot pot + results[4].assert_approx_eq((567, 0.0152781)); // frying pan // This above results almost match (see "off by one" comment above) the output of running // OpenVINO's `hello_classification` with the same inputs: @@ -74,25 +75,3 @@ fn classify_mobilenet() { // 965 0.0058377 // 545 0.0043731 } - -/// A structure for holding the `(category, probability)` pair extracted from the output tensor of -/// the OpenVINO classification. -#[derive(Debug, PartialEq)] -struct Result(usize, f32); -type Results = Vec; - -impl Result { - fn assert_approx_eq(&self, expected: &Result) { - assert_eq!( - self.0, expected.0, - "Expected class ID {} but found {}", - expected.0, self.0 - ); - let approx_matches = approx_eq!(f32, self.1, expected.1, ulps = 2, epsilon = 0.01); - assert!( - approx_matches, - "Expected probability {} but found {} (outside of tolerance)", - expected.1, self.1 - ); - } -} diff --git a/crates/openvino/tests/util.rs b/crates/openvino/tests/util.rs new file mode 100644 index 0000000..a847598 --- /dev/null +++ b/crates/openvino/tests/util.rs @@ -0,0 +1,81 @@ +use core::cmp::Ordering; +use float_cmp::{ApproxEq, F32Margin}; + +/// A structure for holding the `(category, probability)` pair extracted from the output tensor of +/// the OpenVINO classification. +#[derive(Debug)] +pub struct Prediction { + id: usize, + prob: f32, +} + +impl Prediction { + pub fn new(id: usize, prob: f32) -> Self { + Self { id, prob } + } + + /// Reduce the boilerplate to assert that two predictions are approximately the same. + pub fn assert_approx_eq>(&self, expected: P) { + let expected = expected.into(); + assert_eq!( + self.id, expected.id, + "Expected class ID {} but found {}", + expected.id, self.id + ); + let approx_matches = self.approx_eq(&expected, DEFAULT_MARGIN); + assert!( + approx_matches, + "Expected probability {} but found {} (outside of default margin of error)", + expected.prob, self.prob + ); + } +} + +impl From<(usize, f32)> for Prediction { + fn from(p: (usize, f32)) -> Self { + Prediction::new(p.0, p.1) + } +} + +/// Classification results are ordered by their probability, from greatest to smallest. +impl Ord for Prediction { + fn cmp(&self, other: &Self) -> Ordering { + assert!(!self.prob.is_nan()); + assert!(!other.prob.is_nan()); + other + .prob + .partial_cmp(&self.prob) + .expect("a comparable value") + } +} + +impl PartialOrd for Prediction { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for Prediction { + fn eq(&self, other: &Self) -> bool { + self.prob == other.prob + } +} + +impl Eq for Prediction {} + +impl ApproxEq for &Prediction { + type Margin = F32Margin; + fn approx_eq>(self, other: Self, margin: T) -> bool { + let margin = margin.into(); + self.prob.approx_eq(other.prob, margin) + } +} + +/// The default margin for error allowed for comparing classification results. +pub const DEFAULT_MARGIN: F32Margin = F32Margin { + epsilon: 0.01, + ulps: 2, +}; + +/// A helper type for manipulating lists of results. +pub type Predictions = Vec;