From 81b12902cfb492b7fc50ee1f12437c38f6f20491 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 22 Feb 2021 17:39:08 -0500 Subject: [PATCH 1/3] compile engine from SparseZoo stub --- src/deepsparse/engine.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/deepsparse/engine.py b/src/deepsparse/engine.py index a794b2ecd9..41d679e48e 100644 --- a/src/deepsparse/engine.py +++ b/src/deepsparse/engine.py @@ -26,10 +26,15 @@ try: + from sparsezoo import Zoo from sparsezoo.objects import File, Model -except Exception: + + sparsezoo_import_error = None +except Exception as sparsezoo_err: + Zoo = None Model = object File = object + sparsezoo_import_error = sparsezoo_err try: # flake8: noqa @@ -55,8 +60,10 @@ def _model_to_path(model: Union[str, Model, File]) -> str: if not model: raise ValueError("model must be a path, sparsezoo.Model, or sparsezoo.File") - if isinstance(model, str): - pass + if isinstance(model, str) and model.startswith("zoo:"): + if sparsezoo_import_error is not None: + raise sparsezoo_import_error + model = Zoo.load_model_from_stub(model).onnx_file.downloaded_path() elif Model is not object and isinstance(model, Model): # default to the main onnx file for the model model = model.onnx_file.downloaded_path() @@ -105,8 +112,9 @@ class Engine(object): | # create an engine for batch size 1 on all available cores | engine = Engine("path/to/onnx", batch_size=1, num_cores=None) - :param model: Either a path to the model's onnx file, a sparsezoo Model object, - or a sparsezoo ONNX File object that defines the neural network + :param model: Either a path to the model's onnx file, a SparseZoo model stub + prefixed by 'zoo:', a SparseZoo Model object, or a SparseZoo ONNX File + object that defines the neural network :param batch_size: The batch size of the inputs to be used with the engine :param num_cores: The number of physical cores to run the model on. Pass None or 0 to run on the max number of cores @@ -453,8 +461,9 @@ def compile_model( Gives defaults of batch_size == 1 and num_cores == None (will use all physical cores available on a single socket). - :param model: Either a path to the model's onnx file, a sparsezoo Model object, - or a sparsezoo ONNX File object that defines the neural network + :param model: Either a path to the model's onnx file, a SparseZoo model stub + prefixed by 'zoo:', a SparseZoo Model object, or a SparseZoo ONNX File + object that defines the neural network :param batch_size: The batch size of the inputs to be used with the model :param num_cores: The number of physical cores to run the model on. Pass None or 0 to run on the max number of cores @@ -480,8 +489,9 @@ def benchmark_model( Gives defaults of batch_size == 1 and num_cores == None (will use all physical cores available on a single socket). - :param model: Either a path to the model's onnx file, a sparsezoo Model object, - or a sparsezoo ONNX File object that defines the neural network + :param model: Either a path to the model's onnx file, a SparseZoo model stub + prefixed by 'zoo:', a SparseZoo Model object, or a SparseZoo ONNX File + object that defines the neural network :param batch_size: The batch size of the inputs to be used with the model :param num_cores: The number of physical cores to run the model on. Pass None or 0 to run on the max number of cores @@ -524,9 +534,9 @@ def analyze_model( Gives defaults of batch_size == 1 and num_cores == None (will use all physical cores available on a single socket). - :param model: Either a path to the model's onnx file, a sparsezoo Model object, - or a sparsezoo ONNX File object that defines the neural network - graph definition to analyze + :param model: Either a path to the model's onnx file, a SparseZoo model stub + prefixed by 'zoo:', a SparseZoo Model object, or a SparseZoo ONNX File + object that defines the neural network graph definition to analyze :param inp: The list of inputs to pass to the engine for analyzing inference. The expected order is the inputs order as defined in the ONNX graph. :param batch_size: The batch size of the inputs to be used with the model From 50a93daab589d6d717474e81fa1d4ab47c571382 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 22 Feb 2021 20:48:01 -0500 Subject: [PATCH 2/3] optional benchmarking tqdm progress bar --- src/deepsparse/engine.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/deepsparse/engine.py b/src/deepsparse/engine.py index 41d679e48e..79ba4d010f 100644 --- a/src/deepsparse/engine.py +++ b/src/deepsparse/engine.py @@ -23,6 +23,7 @@ import numpy from deepsparse.benchmark import BenchmarkResults +from tqdm.auto import tqdm try: @@ -332,6 +333,7 @@ def benchmark( num_warmup_iterations: int = 5, include_inputs: bool = False, include_outputs: bool = False, + show_progress: bool = False, ) -> BenchmarkResults: """ A convenience function for quickly benchmarking the instantiated model @@ -350,6 +352,7 @@ def benchmark( will be added to the results. Default is False :param include_outputs: If True, outputs from forward passes during benchmarking will be added to the results. Default is False + :param show_progress: If True, will display a progress bar. Default is False :return: the results of benchmarking """ # define data loader @@ -363,6 +366,7 @@ def _infinite_loader(): num_warmup_iterations=num_warmup_iterations, include_inputs=include_inputs, include_outputs=include_outputs, + show_progress=show_progress, ) def benchmark_loader( @@ -372,6 +376,7 @@ def benchmark_loader( num_warmup_iterations: int = 5, include_inputs: bool = False, include_outputs: bool = False, + show_progress: bool = False, ) -> BenchmarkResults: """ A convenience function for quickly benchmarking the instantiated model @@ -390,6 +395,7 @@ def benchmark_loader( will be added to the results. Default is False :param include_outputs: If True, outputs from forward passes during benchmarking will be added to the results. Default is False + :param show_progress: If True, will display a progress bar. Default is False :return: the results of benchmarking """ assert num_iterations >= 1 and num_warmup_iterations >= 0, ( @@ -399,13 +405,15 @@ def benchmark_loader( completed_iterations = 0 results = BenchmarkResults() + if show_progress: + progress_bar = tqdm(total=num_iterations) + while completed_iterations < num_warmup_iterations + num_iterations: for batch in loader: # run benchmark start = time.time() out = self.run(batch) end = time.time() - completed_iterations += 1 if completed_iterations >= num_warmup_iterations: # update results if warmup iterations are completed @@ -416,10 +424,17 @@ def benchmark_loader( inputs=batch if include_inputs else None, outputs=out if include_outputs else None, ) + if show_progress: + progress_bar.update(1) + + completed_iterations += 1 if completed_iterations >= num_warmup_iterations + num_iterations: break + if show_progress: + progress_bar.close() + return results def _validate_inputs(self, inp: List[numpy.ndarray]): @@ -482,6 +497,7 @@ def benchmark_model( num_warmup_iterations: int = 5, include_inputs: bool = False, include_outputs: bool = False, + show_progress: bool = False, ) -> BenchmarkResults: """ Convenience function to benchmark a model in the DeepSparse Engine @@ -508,12 +524,18 @@ def benchmark_model( will be added to the results. Default is False :param include_outputs: If True, outputs from forward passes during benchmarking will be added to the results. Default is False + :param show_progress: If True, will display a progress bar. Default is False :return: the results of benchmarking """ model = compile_model(model, batch_size, num_cores) return model.benchmark( - inp, num_iterations, num_warmup_iterations, include_inputs, include_outputs + inp, + num_iterations, + num_warmup_iterations, + include_inputs, + include_outputs, + show_progress, ) From a9d331bb1a11b1c8584c701cbdcbd40ffc27b96f Mon Sep 17 00:00:00 2001 From: Benjamin Date: Tue, 23 Feb 2021 12:20:57 -0500 Subject: [PATCH 3/3] responding to review --- setup.cfg | 2 +- setup.py | 2 +- src/deepsparse/engine.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 80621586b9..0dbc6fe79a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ ensure_newline_before_comments = True force_grid_wrap = 0 include_trailing_comma = True known_first_party = deepsparse,sparsezoo -known_third_party = numpy,onnx,requests,onnxruntime,flask,flask_cors +known_third_party = numpy,onnx,requests,onnxruntime,flask,flask_cors,tqdm sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER line_length = 88 diff --git a/setup.py b/setup.py index ea9a8594f9..2697b96e9f 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ binary_regexes = ["*/*.so", "*/*.so.*", "*.bin", "*/*.bin"] -_deps = ["numpy>=1.16.3", "onnx>=1.5.0,<1.8.0", "requests>=2.0.0"] +_deps = ["numpy>=1.16.3", "onnx>=1.5.0,<1.8.0", "requests>=2.0.0", "tqdm>=4.0.0"] _nm_deps = [f"{'sparsezoo-nightly' if _NIGHTLY else 'sparsezoo'}~={_VERSION}"] diff --git a/src/deepsparse/engine.py b/src/deepsparse/engine.py index 79ba4d010f..dbe909fdf0 100644 --- a/src/deepsparse/engine.py +++ b/src/deepsparse/engine.py @@ -62,10 +62,12 @@ def _model_to_path(model: Union[str, Model, File]) -> str: raise ValueError("model must be a path, sparsezoo.Model, or sparsezoo.File") if isinstance(model, str) and model.startswith("zoo:"): + # load SparseZoo Model from stub if sparsezoo_import_error is not None: raise sparsezoo_import_error - model = Zoo.load_model_from_stub(model).onnx_file.downloaded_path() - elif Model is not object and isinstance(model, Model): + model = Zoo.load_model_from_stub(model) + + if Model is not object and isinstance(model, Model): # default to the main onnx file for the model model = model.onnx_file.downloaded_path() elif File is not object and isinstance(model, File):