Fast tests (#50)

* smaller sizes

* nb_epoch arg

* reduce num epochs

* don't skip system_query  - doesn't seem to help much

* restore sysingfo

* unset nb_epoch in torch and chainer

* epochs in opencv

benchmarker/benchmarker.py

@@ -23,7 +23,6 @@ def get_modules():
 def run(args, unknown_args):
     params = {}
     params["platform"] = sysinfo.get_sys_info()
-
     if args.framework is None:
         print("please choose one of the frameworks to evaluate")
         print("available frameworks:")

benchmarker/modules/do_chainer.py

@@ -16,7 +16,6 @@ class Benchmark(INeuralNet):
     def __init__(self, params, remaining_args=None):
         super().__init__(params, remaining_args)
         self.params["channels_first"] = True
-        self.params["nb_epoch"] = 10
 
     def do_inference(self, model, x_train, y_train):
         chainer.enable_backprop = False

benchmarker/modules/do_mxnet.py

@@ -12,7 +12,6 @@ class Benchmark(INeuralNet):
 
     def __init__(self, params, unparsed_args):
         super().__init__(params, unparsed_args)
-        self.params["nb_epoch"] = 10
         # TODO: confirm tensor ordering in mxnet
         # self.params["channels_first"] = True
 

benchmarker/modules/do_opencv.py

@@ -9,7 +9,6 @@ class Benchmark(INeuralNet):
     def __init__(self, params, extra_args):
         super().__init__(params, extra_args)
         self.params["channels_first"] = False
-        self.params["nb_epoch"] = 1
         if self.params["mode"] != "inference":
             raise RuntimeError("opencv only supports inference")
         if self.params["batch_size"] != 1:

benchmarker/modules/do_pytorch.py

@@ -24,7 +24,6 @@ def __init__(self, params, extra_args=None):
                 raise RuntimeError("only native backend is supported for GPUs")
 
         self.params["channels_first"] = True
-        self.params["nb_epoch"] = 6
 
     def train(self, model, device, optimizer, epoch):
         model.train()

benchmarker/modules/i_neural_net.py

@@ -12,9 +12,11 @@ def __init__(self, params, extra_args=None):
 
         parser = argparse.ArgumentParser(description="Benchmark deep learning models")
         parser.add_argument("--mode", default="training")
+        parser.add_argument("--nb_epoch", type=int, default=10)
         parsed_args, remaining_args = parser.parse_known_args(extra_args)
 
         params["mode"] = parsed_args.mode
+        params["nb_epoch"] = parsed_args.nb_epoch
         assert params["mode"] in ["training", "inference"]
 
         params["path_out"] = os.path.join(params["path_out"], params["mode"])

test/pytorch/test_bert.py

@@ -17,6 +17,7 @@ def test_bert(self):
             "--problem=bert",
             "--problem_size=32,32",
             "--batch_size=8",
+            "--nb_epoch=1",
             "--mode=inference",
             "--cnt_units=128",
             "--cnt_heads=4",

test/pytorch/test_cnn.py

@@ -14,6 +14,7 @@ def setUp(self):
             "--framework=pytorch",
             "--problem_size=4",
             "--batch_size=2",
+            "--nb_epoch=1",
         ]
 
     def test_cnn2d_inference(self):

test/pytorch/test_conv123d.py

@@ -11,15 +11,16 @@ def setUp(self):
         self.args = [
             "benchmarker",
             "--framework=pytorch",
-            "--batch_size=4",
+            "--batch_size=2",
+            "--nb_epoch=1",
             "--mode=inference",
         ]
 
     def test_conv1d(self):
-        run_module(*self.args + ["--problem=conv1d", "--problem_size=4,4,4"])
+        run_module(*self.args + ["--problem=conv1d", "--problem_size=2,4,4"])
 
     def test_conv2d(self):
-        run_module(*self.args + ["--problem=conv2d", "--problem_size=4,4,4,4"])
+        run_module(*self.args + ["--problem=conv2d", "--problem_size=2,4,4,4"])
 
     def test_conv3d(self):
-        run_module(*self.args + ["--problem=conv3d", "--problem_size=4,4,4,4,4"])
+        run_module(*self.args + ["--problem=conv3d", "--problem_size=2,4,4,4,4"])

test/pytorch/test_lstm.py

@@ -14,5 +14,6 @@ def test_lstm(self):
             "--problem=lstm",
             "--problem_size=4,4,4",
             "--batch_size=4",
+            "--nb_epoch=1",
             "--mode=inference",
         )

test/pytorch/test_resnet50.py

@@ -14,6 +14,7 @@ def setUp(self):
             "--framework=pytorch",
             "--problem_size=4",
             "--batch_size=2",
+            "--nb_epoch=1",
         ]
 
     def test_resnet50(self):

test/pytorch/test_ssd300.py

@@ -16,6 +16,7 @@ def setUp(self):
             "--framework=pytorch",
             "--problem_size=4",
             "--batch_size=2",
+            "--nb_epoch=1",
             "--mode=inference",
         ]
 

test/pytorch/test_vgg16.py

@@ -14,6 +14,7 @@ def setUp(self):
             "--framework=pytorch",
             "--problem_size=4",
             "--batch_size=2",
+            "--nb_epoch=1",
         ]
 
     def test_vgg16(self):

test/tensorflow/test_conv123d.py

@@ -11,15 +11,16 @@ def setUp(self):
         self.args = [
             "benchmarker",
             "--framework=tensorflow",
-            "--batch_size=4",
+            "--batch_size=2",
+            "--nb_epoch=1",
             "--mode=inference",
         ]
 
     def test_conv1d(self):
-        run_module(*self.args + ["--problem=conv1d", "--problem_size=4,4,4"])
+        run_module(*self.args + ["--problem=conv1d", "--problem_size=2,4,4"])
 
     def test_conv2d(self):
-        run_module(*self.args + ["--problem=conv2d", "--problem_size=4,4,4,4"])
+        run_module(*self.args + ["--problem=conv2d", "--problem_size=2,4,4,4"])
 
     def test_conv3d(self):
-        run_module(*self.args + ["--problem=conv3d", "--problem_size=4,4,4,4,4"])
+        run_module(*self.args + ["--problem=conv3d", "--problem_size=2,4,4,4,4"])

test/tensorflow/test_resnet50.py

@@ -13,8 +13,9 @@ def setUp(self):
         self.name = "benchmarker"
         self.imgnet_args = [
             "--framework=tensorflow",
-            "--problem_size=4",
+            "--problem_size=2",
             "--batch_size=2",
+            "--nb_epoch=1",
         ]
 
     def test_resnet50(self):

test/tensorflow/test_vgg16.py

@@ -13,8 +13,9 @@ def setUp(self):
         self.name = "benchmarker"
         self.imgnet_args = [
             "--framework=tensorflow",
-            "--problem_size=4",
+            "--problem_size=2",
             "--batch_size=2",
+            "--nb_epoch=1",
         ]
 
     def test_vgg16(self):