From 5e3f3c74e1a2b92aaa0b0435e1731078651349eb Mon Sep 17 00:00:00 2001 From: mgoin Date: Tue, 26 Apr 2022 21:11:30 -0400 Subject: [PATCH 1/5] Move all benchmarking within deepsparse/benchmark/ --- examples/amd-azure/multi_process_benchmark.py | 2 +- examples/benchmark/check_correctness.py | 2 +- examples/huggingface-transformers/README.md | 2 +- setup.py | 2 +- .../{benchmark_model => benchmark}/README.md | 0 .../{benchmark_model => benchmark}/__init__.py | 0 .../benchmark_model.py | 0 .../img/json_output.png | Bin .../{benchmark_model => benchmark}/ort_engine.py | 0 .../{benchmark.py => benchmark/results.py} | 0 .../stream_benchmark.py | 0 src/deepsparse/engine.py | 2 +- 12 files changed, 5 insertions(+), 5 deletions(-) rename src/deepsparse/{benchmark_model => benchmark}/README.md (100%) rename src/deepsparse/{benchmark_model => benchmark}/__init__.py (100%) rename src/deepsparse/{benchmark_model => benchmark}/benchmark_model.py (100%) rename src/deepsparse/{benchmark_model => benchmark}/img/json_output.png (100%) rename src/deepsparse/{benchmark_model => benchmark}/ort_engine.py (100%) rename src/deepsparse/{benchmark.py => benchmark/results.py} (100%) rename src/deepsparse/{benchmark_model => benchmark}/stream_benchmark.py (100%) diff --git a/examples/amd-azure/multi_process_benchmark.py b/examples/amd-azure/multi_process_benchmark.py index d1b975d137..378764a631 100644 --- a/examples/amd-azure/multi_process_benchmark.py +++ b/examples/amd-azure/multi_process_benchmark.py @@ -23,7 +23,7 @@ import numa from deepsparse import compile_model -from deepsparse.benchmark_model.stream_benchmark import singlestream_benchmark +from deepsparse.benchmark.stream_benchmark import singlestream_benchmark from deepsparse.log import set_logging_level from deepsparse.utils import ( generate_random_inputs, diff --git a/examples/benchmark/check_correctness.py b/examples/benchmark/check_correctness.py index 67d1af429c..59d6a61ae3 100644 --- a/examples/benchmark/check_correctness.py +++ b/examples/benchmark/check_correctness.py @@ -45,7 +45,7 @@ import argparse from deepsparse import compile_model, cpu -from deepsparse.benchmark_model.ort_engine import ORTEngine +from deepsparse.benchmark.ort_engine import ORTEngine from deepsparse.utils import ( generate_random_inputs, model_to_path, diff --git a/examples/huggingface-transformers/README.md b/examples/huggingface-transformers/README.md index 4a54903cdc..863da102a8 100644 --- a/examples/huggingface-transformers/README.md +++ b/examples/huggingface-transformers/README.md @@ -89,7 +89,7 @@ Want to find out how fast our sparse Hugging Face ONNX models perform inference? deepsparse.benchmark zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/12layer_pruned80_quant-none-vnni ``` -For a more in-depth discussion on benchmarking, check out the [Benchmarking tutorial](https://github.com/neuralmagic/deepsparse/tree/main/src/deepsparse/benchmark_model)! +For a more in-depth discussion on benchmarking, check out the [Benchmarking tutorial](https://github.com/neuralmagic/deepsparse/tree/main/src/deepsparse/benchmark)! ### 🔌 DeepSparse Server diff --git a/setup.py b/setup.py index 855f119dd5..94d8cfc70c 100644 --- a/setup.py +++ b/setup.py @@ -185,7 +185,7 @@ def _setup_entry_points() -> Dict: f"deepsparse.transformers.eval_downstream={eval_downstream}", "deepsparse.analyze=deepsparse.analyze:main", "deepsparse.check_hardware=deepsparse.cpu:print_hardware_capability", - "deepsparse.benchmark=deepsparse.benchmark_model.benchmark_model:main", + "deepsparse.benchmark=deepsparse.benchmark.benchmark_model:main", "deepsparse.server=deepsparse.server.main:start_server", ] } diff --git a/src/deepsparse/benchmark_model/README.md b/src/deepsparse/benchmark/README.md similarity index 100% rename from src/deepsparse/benchmark_model/README.md rename to src/deepsparse/benchmark/README.md diff --git a/src/deepsparse/benchmark_model/__init__.py b/src/deepsparse/benchmark/__init__.py similarity index 100% rename from src/deepsparse/benchmark_model/__init__.py rename to src/deepsparse/benchmark/__init__.py diff --git a/src/deepsparse/benchmark_model/benchmark_model.py b/src/deepsparse/benchmark/benchmark_model.py similarity index 100% rename from src/deepsparse/benchmark_model/benchmark_model.py rename to src/deepsparse/benchmark/benchmark_model.py diff --git a/src/deepsparse/benchmark_model/img/json_output.png b/src/deepsparse/benchmark/img/json_output.png similarity index 100% rename from src/deepsparse/benchmark_model/img/json_output.png rename to src/deepsparse/benchmark/img/json_output.png diff --git a/src/deepsparse/benchmark_model/ort_engine.py b/src/deepsparse/benchmark/ort_engine.py similarity index 100% rename from src/deepsparse/benchmark_model/ort_engine.py rename to src/deepsparse/benchmark/ort_engine.py diff --git a/src/deepsparse/benchmark.py b/src/deepsparse/benchmark/results.py similarity index 100% rename from src/deepsparse/benchmark.py rename to src/deepsparse/benchmark/results.py diff --git a/src/deepsparse/benchmark_model/stream_benchmark.py b/src/deepsparse/benchmark/stream_benchmark.py similarity index 100% rename from src/deepsparse/benchmark_model/stream_benchmark.py rename to src/deepsparse/benchmark/stream_benchmark.py diff --git a/src/deepsparse/engine.py b/src/deepsparse/engine.py index f1ba92d8e4..33a8ae6f52 100644 --- a/src/deepsparse/engine.py +++ b/src/deepsparse/engine.py @@ -23,7 +23,7 @@ import numpy from tqdm.auto import tqdm -from deepsparse.benchmark import BenchmarkResults +from deepsparse.benchmark.results import BenchmarkResults from deepsparse.utils import model_to_path, override_onnx_input_shapes From 6cde14af4e5f2c2f626084e31ea520c36066127a Mon Sep 17 00:00:00 2001 From: mgoin Date: Wed, 27 Apr 2022 09:35:42 -0400 Subject: [PATCH 2/5] Update benchmark_model --- src/deepsparse/benchmark/benchmark_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/deepsparse/benchmark/benchmark_model.py b/src/deepsparse/benchmark/benchmark_model.py index 4ff874b0c6..caed1a5c28 100644 --- a/src/deepsparse/benchmark/benchmark_model.py +++ b/src/deepsparse/benchmark/benchmark_model.py @@ -97,8 +97,8 @@ import os from deepsparse import Scheduler, compile_model -from deepsparse.benchmark_model.ort_engine import ORTEngine -from deepsparse.benchmark_model.stream_benchmark import model_stream_benchmark +from deepsparse.benchmark.ort_engine import ORTEngine +from deepsparse.benchmark.stream_benchmark import model_stream_benchmark from deepsparse.log import set_logging_level from deepsparse.utils import ( generate_random_inputs, From cede8abdfbf21382d1fb628fea2804e636801495 Mon Sep 17 00:00:00 2001 From: mgoin Date: Wed, 27 Apr 2022 09:39:34 -0400 Subject: [PATCH 3/5] Expose results at benchmark base --- examples/benchmark/run_benchmark.py | 6 +++--- src/deepsparse/benchmark/__init__.py | 4 ++++ src/deepsparse/engine.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/benchmark/run_benchmark.py b/examples/benchmark/run_benchmark.py index 0f3b487155..43d33be815 100644 --- a/examples/benchmark/run_benchmark.py +++ b/examples/benchmark/run_benchmark.py @@ -145,13 +145,13 @@ def main(): inputs, num_iterations, num_warmup_iterations, include_outputs=True ) - for dse_output, ort_output in zip(dse_results.outputs, ort_results.outputs): - verify_outputs(dse_output, ort_output) - print("ONNXRuntime", ort_results) print() print("DeepSparse Engine", dse_results) + for dse_output, ort_output in zip(dse_results.outputs, ort_results.outputs): + verify_outputs(dse_output, ort_output) + if __name__ == "__main__": main() diff --git a/src/deepsparse/benchmark/__init__.py b/src/deepsparse/benchmark/__init__.py index 0c44f887a4..3b1a336f04 100644 --- a/src/deepsparse/benchmark/__init__.py +++ b/src/deepsparse/benchmark/__init__.py @@ -11,3 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from .results import * + +from .ort_engine import ORTEngine \ No newline at end of file diff --git a/src/deepsparse/engine.py b/src/deepsparse/engine.py index 33a8ae6f52..f1ba92d8e4 100644 --- a/src/deepsparse/engine.py +++ b/src/deepsparse/engine.py @@ -23,7 +23,7 @@ import numpy from tqdm.auto import tqdm -from deepsparse.benchmark.results import BenchmarkResults +from deepsparse.benchmark import BenchmarkResults from deepsparse.utils import model_to_path, override_onnx_input_shapes From 367799d02fcfc0e1f587affcf326e08fba2ae940 Mon Sep 17 00:00:00 2001 From: mgoin Date: Wed, 27 Apr 2022 09:41:31 -0400 Subject: [PATCH 4/5] isort --- src/deepsparse/benchmark/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/deepsparse/benchmark/__init__.py b/src/deepsparse/benchmark/__init__.py index 3b1a336f04..fede977e83 100644 --- a/src/deepsparse/benchmark/__init__.py +++ b/src/deepsparse/benchmark/__init__.py @@ -12,6 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .ort_engine import ORTEngine from .results import * - -from .ort_engine import ORTEngine \ No newline at end of file From b2ac821546fe41ce29b377e85cc687d5c2cf4929 Mon Sep 17 00:00:00 2001 From: mgoin Date: Wed, 27 Apr 2022 09:45:27 -0400 Subject: [PATCH 5/5] Skip flake8 --- src/deepsparse/benchmark/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/deepsparse/benchmark/__init__.py b/src/deepsparse/benchmark/__init__.py index fede977e83..4900c1c550 100644 --- a/src/deepsparse/benchmark/__init__.py +++ b/src/deepsparse/benchmark/__init__.py @@ -12,5 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# flake8: noqa + from .ort_engine import ORTEngine from .results import *