Merge branch 'master' into ncf-WIP

README.rst

@@ -25,7 +25,7 @@ Benchmarker is a modular framework to automate a set of performance benchmarks,
 Run
 ===
 
-python3 -m benchmarker  --mode=training --framework=chainer --problem=resnet50 --problem_size=32 --batch_size=4
+python3 -m benchmarker  --mode=training --framework=pytorch --problem=resnet50 --problem_size=32 --batch_size=4
 
 
 ==========

benchmarker/__main__.py

@@ -1,22 +1,13 @@
 """CLI entry point module"""
 
-import argparse
+import sys
+
 from .benchmarker import run
 
 
 def main():
     """CLI entry point function"""
-    parser = argparse.ArgumentParser(description='Benchmark me up, Scotty!')
-    parser.add_argument("--framework")
-    parser.add_argument("--problem")
-    parser.add_argument('--path_out', type=str, default="./logs")
-    parser.add_argument('--gpus', default="")
-    parser.add_argument('--problem_size', default=None)
-    parser.add_argument('--batch_size', default=None)
-    # parser.add_argument('--misc')
-    # TODO: move this inside benchmarker class
-    args, unknown_args = parser.parse_known_args()
-    run(args, unknown_args)
+    run(sys.argv[1:])
 
 
 if __name__ == "__main__":

benchmarker/_version.py

@@ -1,3 +1,3 @@
 """Version of benchamrker package."""
 
-VERSION = "0.1.7"
+VERSION = "0.2.0"

benchmarker/benchmarker.py

@@ -4,8 +4,9 @@
 This is where all magic is happening
 """
 
-import importlib
+import argparse
 import ast
+import importlib
 import pkgutil
 # import logging
 import os
@@ -20,10 +21,22 @@ def get_modules():
             if not is_pkg and name.startswith('do_')]
 
 
-def run(args, unknown_args):
+def parse_basic_args(argv):
+    parser = argparse.ArgumentParser(description='Benchmark me up, Scotty!')
+    parser.add_argument("--framework")
+    parser.add_argument("--problem")
+    parser.add_argument('--path_out', type=str, default="./logs")
+    parser.add_argument('--gpus', default="")
+    parser.add_argument('--problem_size', default=None)
+    parser.add_argument('--batch_size', default=None)
+    # parser.add_argument('--misc')
+    return parser.parse_known_args(argv)
+
+
+def run(argv):
+    args, unknown_args = parse_basic_args(argv)
     params = {}
     params["platform"] = sysinfo.get_sys_info()
-
     if args.framework is None:
         print("please choose one of the frameworks to evaluate")
         print("available frameworks:")
@@ -52,8 +65,10 @@ def run(args, unknown_args):
         params["batch_size"] = int(args.batch_size)
         params["batch_size_per_device"] = int(args.batch_size)
     # params["misc"] = args.misc
+    os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
     if args.gpus:
-        params["gpus"] = list(map(int, args.gpus.split(',')))
+        os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus
+        params["gpus"] = list(map(int, args.gpus.split(",")))
     else:
         params["gpus"] = []
 

benchmarker/modules/do_chainer.py

@@ -16,7 +16,6 @@ class Benchmark(INeuralNet):
     def __init__(self, params, remaining_args=None):
         super().__init__(params, remaining_args)
         self.params["channels_first"] = True
-        self.params["nb_epoch"] = 10
 
     def do_inference(self, model, x_train, y_train):
         chainer.enable_backprop = False

benchmarker/modules/do_mxnet.py

@@ -12,7 +12,6 @@ class Benchmark(INeuralNet):
 
     def __init__(self, params, unparsed_args):
         super().__init__(params, unparsed_args)
-        self.params["nb_epoch"] = 10
         # TODO: confirm tensor ordering in mxnet
         # self.params["channels_first"] = True
 

benchmarker/modules/do_opencv.py

@@ -9,7 +9,6 @@ class Benchmark(INeuralNet):
     def __init__(self, params, extra_args):
         super().__init__(params, extra_args)
         self.params["channels_first"] = False
-        self.params["nb_epoch"] = 1
         if self.params["mode"] != "inference":
             raise RuntimeError("opencv only supports inference")
         if self.params["batch_size"] != 1:

benchmarker/modules/do_pytorch.py

@@ -1,21 +1,30 @@
-import numpy as np
+import argparse
+from timeit import default_timer as timer
+
 import torch
+# TODO: should we expect an import error here?
+# https://stackoverflow.com/questions/3496592/conditional-import-of-modules-in-python
+import torch.backends.mkldnn
 import torch.nn as nn
-from timeit import default_timer as timer
 # import torch.nn.functional as F
 import torch.optim as optim
+from torch.utils import mkldnn as mkldnn_utils
+
 # from torchvision import datasets, transforms
 from .i_neural_net import INeuralNet
-import argparse
-# TODO: should we expect an import error here?
-# https://stackoverflow.com/questions/3496592/conditional-import-of-modules-in-python
-import torch.backends.mkldnn
+
+
+def progress(epoch, idx, nb, loss, log_interval=10):
+    if idx % log_interval == 0:
+        prc = 100.0 * idx / nb
+        stat = f"{epoch} [{idx}/{nb} ({prc:.0f}%)]\tLoss: {loss:.6f}"
+        print("Train Epoch: " + stat)
 
 
 class Benchmark(INeuralNet):
     def __init__(self, params, extra_args=None):
-        parser = argparse.ArgumentParser(description='pytorch extra args')
-        parser.add_argument('--backend', default="native")
+        parser = argparse.ArgumentParser(description="pytorch extra args")
+        parser.add_argument("--backend", default="native")
         args, remaining_args = parser.parse_known_args(extra_args)
         super().__init__(params, remaining_args)
         self.params["backend"] = args.backend
@@ -24,32 +33,29 @@ def __init__(self, params, extra_args=None):
                 raise RuntimeError("only native backend is supported for GPUs")
 
         self.params["channels_first"] = True
-        self.params["nb_epoch"] = 6
 
     def train(self, model, device, optimizer, epoch):
         model.train()
         for batch_idx, (data, target) in enumerate(zip(self.x_train, self.y_train)):
             optimizer.zero_grad()
             loss = model(data, target)
-            # loss = F.nll_loss(output, target)
-            # TODO: criterion should be included in the model, deepening on params
-            #criterion = nn.CrossEntropyLoss()
-            #loss = criterion(output, target)
-            loss.backward()
+            loss.mean().backward()
             optimizer.step()
-            log_interval = 10
-            if batch_idx % log_interval == 0:
-                print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
-                    epoch, batch_idx * len(data), len(self.x_train),
-                    100. * batch_idx / len(self.x_train), loss.item()))
+            progress(epoch, batch_idx, len(self.x_train), loss.mean().item())
         if device.type == "cuda":
             torch.cuda.synchronize()
 
+    def set_random_seed(self, seed):
+        super().set_random_seed(seed)
+        torch.manual_seed(seed)
+
     def inference(self, model, device):
         # test_loss = 0
         # correct = 0
         with torch.no_grad():
             for data, target in zip(self.x_train, self.y_train):
+                if self.params["backend"] == "DNNL":
+                    data = data.to_mkldnn()
                 # TODO: add option to keep data on GPU
                 # data, target = data.to(device), target.to(device)
                 output = model(data)
@@ -61,7 +67,7 @@ def inference(self, model, device):
             torch.cuda.synchronize()
         # test_loss /= len(test_loader.dataset)
 
-        #print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
+        # print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
         #    test_loss, correct, len(test_loader.dataset),
         #    100. * correct / len(test_loader.dataset)))
 
@@ -74,34 +80,34 @@ def run_internal(self):
                 torch.backends.mkldnn.enabled = False
             else:
                 raise RuntimeError("Unknown backend")
-        if self.params["nb_gpus"] > 1:
-            raise NotADirectoryError("multyple GPUs not supported yet")
-        if self.params["gpus"]:
-            torch.cuda.set_device(self.params["gpus"][0])
-            device = torch.device("cuda")
-        else:
-            device = torch.device("cpu")
+        device = torch.device("cuda" if self.params["gpus"] else "cpu")
 
         x_train, y_train = self.load_data()
-        # TODO: make of/on-core optional
-        self.x_train = torch.from_numpy(x_train).to(device)
-        self.y_train = torch.from_numpy(y_train.astype(np.int64)).to(device)
+
         # train_dataset = torch.utils.data.TensorDataset(x_train, y_train)
         # train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=self.params["batch_size"], shuffle=False)
 
-        model = self.net.to(device)
+        model = self.net
+        if len(self.params["gpus"]) > 1:
+            model = nn.DataParallel(model)
+        # TODO: make of/on-core optional
+        self.x_train = torch.from_numpy(x_train).to(device)
+        self.y_train = torch.from_numpy(y_train).to(device)
+        model.to(device)
         # TODO: args for training hyperparameters
-        optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.95)
         start = timer()
         if self.params["mode"] == "training":
+            model.train()
+            optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.95)
             for epoch in range(1, self.params["nb_epoch"] + 1):
                 self.train(model, device, optimizer, epoch)
             # test(args, model, device, test_loader)
         else:
             model.eval()
+            if self.params["backend"] == "DNNL":
+                model = mkldnn_utils.to_mkldnn(model)
             for epoch in range(1, self.params["nb_epoch"] + 1):
                 self.inference(model, device)
-        # TODO: return stats
         end = timer()
         self.params["time"] = (end - start) / self.params["nb_epoch"]
         self.params["framework_full"] = "PyTorch-" + torch.__version__

benchmarker/modules/do_tensorflow.py

@@ -14,16 +14,16 @@ class Benchmark(INeuralNet):
     """docstring for ClassName"""
 
     def __init__(self, params, remaining_args=None):
-        if params["nb_gpus"] < 1:
-            os.environ["CUDA_VISIBLE_DEVICES"] = ""
-        if params["nb_gpus"] > 1:
-            print("multiple gpus with TF not supported yet")
-            return
+        gpus = params["gpus"]
         super().__init__(params, remaining_args)
         self.params["channels_first"] = False
         os.environ["KERAS_BACKEND"] = "tensorflow"
 
     def get_strategy(self):
+        gpu_count_same = self.params["nb_gpus"] == len(
+            tf.config.list_physical_devices("GPU")
+        )
+        assert gpu_count_same, "Tensorflow not compiled with GPU support"
         try:
             tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection
         except ValueError:
@@ -40,13 +40,10 @@ def get_strategy(self):
                 "worker_srt": worker_str,
                 "num_replicas_in_sync": rep,
             }
-        elif len(gpus) > 1 and self.params["nb_gpus"] > 0:
+        elif len(gpus) > 1:
             strategy = tf.distribute.MirroredStrategy(gpus)
         elif self.params["nb_gpus"] == 1:
-            if tf.test.gpu_device_name():
-                strategy = tf.distribute.get_strategy()
-            else:
-                raise RuntimeError("No GPU found")
+            strategy = tf.distribute.get_strategy()
         else:  # Make sure we run on CPU
             strategy = tf.distribute.get_strategy()
 
@@ -60,6 +57,10 @@ def get_kernel(self, module, remaining_args):
         with self.get_strategy().scope():
             super().get_kernel(module, remaining_args)
 
+    def set_random_seed(self, seed):
+        super().set_random_seed(seed)
+        tf.random.set_seed(seed)
+
     def run_internal(self):
 
         # if params["channels_first"]:
@@ -81,7 +82,7 @@ def run_internal(self):
             cnt_classes = 1
         self.params["cnt_classes"] = cnt_classes
         model = self.net
-        nb_epoch = 3
+        nb_epoch = self.params["nb_epoch"]
         bs = self.params["batch_size"]
         if self.params["mode"] == "training":
             print("preheat")

benchmarker/modules/i_neural_net.py

@@ -2,6 +2,9 @@
 import argparse
 import importlib
 import os
+import random
+
+import numpy
 
 
 class INeuralNet:
@@ -12,9 +15,15 @@ def __init__(self, params, extra_args=None):
 
         parser = argparse.ArgumentParser(description="Benchmark deep learning models")
         parser.add_argument("--mode", default="training")
+        parser.add_argument("--nb_epoch", type=int, default=10)
+
+        #
+        parser.add_argument("--random_seed", default=None)
+
         parsed_args, remaining_args = parser.parse_known_args(extra_args)
 
         params["mode"] = parsed_args.mode
+        params["nb_epoch"] = parsed_args.nb_epoch
         assert params["mode"] in ["training", "inference"]
 
         params["path_out"] = os.path.join(params["path_out"], params["mode"])
@@ -33,6 +42,8 @@ def __init__(self, params, extra_args=None):
         self.params["channels_first"] = True
         path = f"benchmarker.modules.problems.{params['problem']['name']}.{params['framework']}"
         kernel_module = importlib.import_module(path)
+        if parsed_args.random_seed is not None:
+            self.set_random_seed(int(parsed_args.random_seed))
         self.get_kernel(kernel_module, remaining_args)
 
     def get_kernel(self, module, remaining_args):
@@ -61,6 +72,17 @@ def load_data(self):
         params["problem"]["size_sample"] = data[0][0].shape
         return data
 
+    def set_random_seed(self, seed):
+        """Default function to set random seeds which sets numpy and random
+        modules seed.  This function should be overridden in the
+        derived classes where this function should be called trough
+        `super()` and also set the random seed of the framework.
+
+        """
+        # os.environ['PYTHONHASHSEED'] = '0' # this seems like too much
+        numpy.random.seed(seed)
+        random.seed(seed)
+
     def run(self):
         results = self.run_internal()
         results["time_epoch"] = results["time"]

benchmarker/modules/problems/bert/data.py

@@ -2,15 +2,15 @@
 
 
 def get_data(params):
-    """generates sinthetic dataset"""
-    #input size -> cnt_sequences, len_suqence, cnt_dimentsions
-    # transform into (seq_len, batch) x cnt_batches
+    if isinstance(params["problem"]["size"], int):
+        params["problem"]["size"] = (params["problem"]["size"], 128)
     assert params["problem"]["size"][0] % params["batch_size"] == 0
     params["problem"]["len_sequence"] = params["problem"]["size"][1]
     cnt_batches = params["problem"]["size"][0] // params["batch_size"]
     shape = (cnt_batches,
+             params["batch_size"],
              params["problem"]["len_sequence"],
-             params["batch_size"])
+             )
     X = np.random.random(shape).astype(np.int64)
-    Y = np.ones((cnt_batches, params["batch_size"]))
+    Y = np.ones((cnt_batches, params["batch_size"]), dtype=np.int64)
     return X, Y