From b0083604527c7c930de1611f81efd29dbbcbacab Mon Sep 17 00:00:00 2001 From: xadupre Date: Tue, 14 Jun 2022 12:38:12 +0200 Subject: [PATCH 1/3] documentation --- .../examples/plot_benchmark_onnx_serialize.py | 105 ++++++++++++++++++ .../source/api/onnx_python/serialize.rst | 7 ++ 2 files changed, 112 insertions(+) create mode 100644 _doc/examples/plot_benchmark_onnx_serialize.py diff --git a/_doc/examples/plot_benchmark_onnx_serialize.py b/_doc/examples/plot_benchmark_onnx_serialize.py new file mode 100644 index 00000000..78178c57 --- /dev/null +++ b/_doc/examples/plot_benchmark_onnx_serialize.py @@ -0,0 +1,105 @@ +""" +.. _l-benchmark-onnx-serialize: + +SerializeToString and ParseFromString +===================================== + + +.. contents:: + :local: + +Startup ++++++++ + +This section creates an ONNX graph if there is not one. + +""" +import numpy +import onnx +from cpyquickhelper.numbers.speed_measure import measure_time +import matplotlib.pyplot as plt +import pandas +from tqdm import tqdm +from mlprodict.testing.experimental_c_impl.experimental_c import code_optimisation +from mlprodict.plotting.text_plot import onnx_simple_text_plot +from mlprodict.npy.xop import loadop + + +############################################ +# Available optimisation on this machine. + +print(code_optimisation()) + + +############################## +# Build an ONNX graph of different size +# +++++++++++++++++++++++++++++++++++++ + +def build_model(n_nodes, size, opv=15): + OnnxAdd, OnnxIdentity = loadop('Add', 'Identity') + x = 'X' + for n in range(n_nodes): + y = OnnxAdd(x, numpy.random.randn(size).astype(numpy.float32), + op_version=opv) + x = y + final = OnnxIdentity(x, op_version=opv, output_names=['Y']) + return final.to_onnx(numpy.float32, numpy.float32, target_opset=opv) + + +model = build_model(2, 5) +print(onnx_simple_text_plot(model)) + +########################################## +# Measure the time of serialization functions +# +++++++++++++++++++++++++++++++++++++++++++ + +def parse(buffer): + proto = onnx.ModelProto() + proto.ParseFromString(buffer) + return proto + +data = [] +nodes = [5, 10, 20] +for size in tqdm([10, 100, 1000, 10000, 100000, 200000, 300000]): + for n_nodes in nodes: + onx = build_model(n_nodes, size) + serialized = onx.SerializeToString() + onnx_size = len(serialized) + obs = measure_time(lambda: onx.SerializeToString(), div_by_number=True, repeat=20) + obs['size'] = size + obs['n_nodes'] = n_nodes + obs['onnx_size'] = onnx_size + obs['task'] = "SerializeToString" + data.append(obs) + + parsed = parse(serialized) + obs = measure_time(lambda: parse(serialized), div_by_number=True, repeat=20) + obs['size'] = size + obs['n_nodes'] = n_nodes + obs['onnx_size'] = onnx_size + obs['task'] = "ParseFromString" + data.append(obs) + + +df = pandas.DataFrame(data).sort_values(['task', 'onnx_size', 'size', 'n_nodes']) +df[['task', 'onnx_size', 'size', 'n_nodes', 'average']] + + +############################################### +# Summary +# +++++++ + +piv = df.pivot(index='onnx_size', columns='task', values='average') +piv + +######################################### +# Graph +# +++++ + +fig, ax = plt.subplots(1, 1) +piv.plot(title="Time processing of serialization functions\n" + "lower better", ax=ax) +ax.set_xlabel("onnx size") +ax.set_ylabel("s") + +plt.show() diff --git a/_doc/sphinxdoc/source/api/onnx_python/serialize.rst b/_doc/sphinxdoc/source/api/onnx_python/serialize.rst index a4d03ba5..53c7b109 100644 --- a/_doc/sphinxdoc/source/api/onnx_python/serialize.rst +++ b/_doc/sphinxdoc/source/api/onnx_python/serialize.rst @@ -65,3 +65,10 @@ an array of bytes. .. autoclass:: onnx.TensorProto :members: SerializeToString + +Performance ++++++++++++ + +.. toctree:: + + ../../gyexamples/plot_benchmark_onnx_serialize From d05eac4fe92eae92c8b31f39d417fae7d6be613e Mon Sep 17 00:00:00 2001 From: xadupre Date: Tue, 14 Jun 2022 12:41:52 +0200 Subject: [PATCH 2/3] lint --- _doc/examples/plot_benchmark_onnx_serialize.py | 11 ++++++++--- _doc/sphinxdoc/source/api/onnx_python/utils.rst | 2 +- .../source/api/onnxruntime_python/inference.rst | 1 - 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/_doc/examples/plot_benchmark_onnx_serialize.py b/_doc/examples/plot_benchmark_onnx_serialize.py index 78178c57..e86379c9 100644 --- a/_doc/examples/plot_benchmark_onnx_serialize.py +++ b/_doc/examples/plot_benchmark_onnx_serialize.py @@ -53,11 +53,13 @@ def build_model(n_nodes, size, opv=15): # Measure the time of serialization functions # +++++++++++++++++++++++++++++++++++++++++++ + def parse(buffer): proto = onnx.ModelProto() proto.ParseFromString(buffer) return proto + data = [] nodes = [5, 10, 20] for size in tqdm([10, 100, 1000, 10000, 100000, 200000, 300000]): @@ -65,7 +67,8 @@ def parse(buffer): onx = build_model(n_nodes, size) serialized = onx.SerializeToString() onnx_size = len(serialized) - obs = measure_time(lambda: onx.SerializeToString(), div_by_number=True, repeat=20) + obs = measure_time(lambda: onx.SerializeToString(), + div_by_number=True, repeat=20) obs['size'] = size obs['n_nodes'] = n_nodes obs['onnx_size'] = onnx_size @@ -73,7 +76,8 @@ def parse(buffer): data.append(obs) parsed = parse(serialized) - obs = measure_time(lambda: parse(serialized), div_by_number=True, repeat=20) + obs = measure_time(lambda: parse(serialized), + div_by_number=True, repeat=20) obs['size'] = size obs['n_nodes'] = n_nodes obs['onnx_size'] = onnx_size @@ -81,7 +85,8 @@ def parse(buffer): data.append(obs) -df = pandas.DataFrame(data).sort_values(['task', 'onnx_size', 'size', 'n_nodes']) +df = pandas.DataFrame(data).sort_values( + ['task', 'onnx_size', 'size', 'n_nodes']) df[['task', 'onnx_size', 'size', 'n_nodes', 'average']] diff --git a/_doc/sphinxdoc/source/api/onnx_python/utils.rst b/_doc/sphinxdoc/source/api/onnx_python/utils.rst index 77ca4e64..80e0dbcf 100644 --- a/_doc/sphinxdoc/source/api/onnx_python/utils.rst +++ b/_doc/sphinxdoc/source/api/onnx_python/utils.rst @@ -44,4 +44,4 @@ dimension .. autofunction:: onnx.compose.expand_out_dim_graph -.. autofunction:: onnx.tools.update_model_dims \ No newline at end of file +.. autofunction:: onnx.tools.update_model_dims diff --git a/_doc/sphinxdoc/source/api/onnxruntime_python/inference.rst b/_doc/sphinxdoc/source/api/onnxruntime_python/inference.rst index 910a221d..826f9b28 100644 --- a/_doc/sphinxdoc/source/api/onnxruntime_python/inference.rst +++ b/_doc/sphinxdoc/source/api/onnxruntime_python/inference.rst @@ -83,4 +83,3 @@ GraphOptimizationLevel .. autoclass:: onnxruntime.capi._pybind_state.GraphOptimizationLevel :members: - From 6debdb38e43782861c036cbf09d545cb5c080bea Mon Sep 17 00:00:00 2001 From: xadupre Date: Tue, 14 Jun 2022 13:36:12 +0200 Subject: [PATCH 3/3] Update plot_benchmark_onnx_serialize.py --- _doc/examples/plot_benchmark_onnx_serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_doc/examples/plot_benchmark_onnx_serialize.py b/_doc/examples/plot_benchmark_onnx_serialize.py index e86379c9..35c20ff3 100644 --- a/_doc/examples/plot_benchmark_onnx_serialize.py +++ b/_doc/examples/plot_benchmark_onnx_serialize.py @@ -107,4 +107,4 @@ def parse(buffer): ax.set_xlabel("onnx size") ax.set_ylabel("s") -plt.show() +# plt.show()