From 0f66ea70c6d78efe988a34090c510f1e8d85b741 Mon Sep 17 00:00:00 2001 From: Flax Team Date: Thu, 16 Jul 2020 10:27:19 -0700 Subject: [PATCH] Remove expectations on secs_per_epoch metric. Technically the tests can be run on any platform and the expectations might change resulting in tests failing. We can introduce such assertions when we integrate the benchmarks with a CI or Regression analyzer. We do export these metrics, so still the external integration without any change should be possible. PiperOrigin-RevId: 321594202 --- examples/cifar10/train_benchmark.py | 1 - examples/imagenet/train_benchmark.py | 1 - examples/mnist/train_benchmark.py | 1 - 3 files changed, 3 deletions(-) diff --git a/examples/cifar10/train_benchmark.py b/examples/cifar10/train_benchmark.py index 0d0076716..4ab9fc832 100644 --- a/examples/cifar10/train_benchmark.py +++ b/examples/cifar10/train_benchmark.py @@ -58,7 +58,6 @@ def test_1x_v100(self): # Assertions are deferred until the test finishes, so the metrics are # always reported and benchmark success is determined based on *all* # assertions. - self.assertBetween(sec_per_epoch, 80., 84.) self.assertBetween(end_error_rate, 0.30, 0.36) # Use the reporting API to report single or multiple metrics/extras. diff --git a/examples/imagenet/train_benchmark.py b/examples/imagenet/train_benchmark.py index 3230f5910..e87a19346 100644 --- a/examples/imagenet/train_benchmark.py +++ b/examples/imagenet/train_benchmark.py @@ -58,7 +58,6 @@ def test_8x_v100_half_precision(self): # Assertions are deferred until the test finishes, so the metrics are # always reported and benchmark success is determined based on *all* # assertions. - self.assertBetween(sec_per_epoch, 210, 240) self.assertBetween(end_accuracy, 0.06, 0.09) # Use the reporting API to report single or multiple metrics/extras. diff --git a/examples/mnist/train_benchmark.py b/examples/mnist/train_benchmark.py index 307a4e94b..5203a408a 100644 --- a/examples/mnist/train_benchmark.py +++ b/examples/mnist/train_benchmark.py @@ -55,7 +55,6 @@ def test_cpu(self): # Assertions are deferred until the test finishes, so the metrics are # always reported and benchmark success is determined based on *all* # assertions. - self.assertBetween(sec_per_epoch, 14., 16.) self.assertBetween(end_eval_accuracy, 0.98, 1.0) # Use the reporting API to report single or multiple metrics/extras.