From 2b0eade218315a9783e7b9ded50f9ed5b2f71713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Sat, 5 Mar 2022 13:46:23 +0100 Subject: [PATCH 1/2] Adds code to turn onnx example into python unit test --- _unittests/ut_testing/test_onnx_backend.py | 130 ++++++++++++++++++++- mlprodict/testing/onnx_backend.py | 42 +++++++ 2 files changed, 169 insertions(+), 3 deletions(-) diff --git a/_unittests/ut_testing/test_onnx_backend.py b/_unittests/ut_testing/test_onnx_backend.py index 779a322a0..e4970a46e 100644 --- a/_unittests/ut_testing/test_onnx_backend.py +++ b/_unittests/ut_testing/test_onnx_backend.py @@ -2,6 +2,12 @@ @brief test log(time=10s) """ import unittest +import numpy +from numpy import array, float32 +from onnx import numpy_helper, TensorProto +from onnx.helper import ( + make_model, make_node, set_model_props, make_tensor, make_graph, + make_tensor_value_info) from pyquickhelper.pycode import ExtTestCase from mlprodict.testing.onnx_backend import enumerate_onnx_tests from mlprodict.onnxrt import OnnxInference @@ -61,11 +67,129 @@ def test_enumerate_onnx_tests_run(self): if __name__ == '__main__': print(len(missed), len(failed), len(mismatch)) for t in failed: - print("failed", t[0]) + print("failed", str(t[0]).replace('\\\\', '\\')) for t in mismatch: - print("mismatch", t[0]) + print("mismatch", str(t[0]).replace('\\\\', '\\')) for t in missed: - print("missed", t[0]) + print("missed", str(t[0]).replace('\\\\', '\\')) + + def test_onnx_backend_test_to_python(self): + name = 'test_abs' + code = [] + for te in enumerate_onnx_tests('node', lambda folder: folder == name): + code.append(te.to_python()) + self.assertEqual(len(code), 1) + self.assertIn('def test_abs(self):', code[0]) + self.assertIn('from onnx.helper', code[0]) + self.assertIn('for y, gy in zip(ys, goty):', code[0]) + if __name__ == '__main__': + print(code[0]) + + def test_abs(self): + + def create_model(): + ''' + Converted ``test_abs``. + + * producer: backend-test + * version: 0 + * description: + ''' + + initializers = [] + nodes = [] + inputs = [] + outputs = [] + + opsets = {'': 9} + target_opset = 9 # subgraphs + + value = make_tensor_value_info('x', 1, [3, 4, 5]) + inputs.append(value) + + value = make_tensor_value_info('y', 1, [3, 4, 5]) + outputs.append(value) + + node = make_node( + 'Abs', + ['x'], + ['y'], + domain='') + nodes.append(node) + + graph = make_graph(nodes, 'test_abs', inputs, + outputs, initializers) + + onnx_model = make_model(graph) + onnx_model.ir_version = 3 + onnx_model.producer_name = 'backend-test' + onnx_model.producer_version = '' + onnx_model.domain = '' + onnx_model.model_version = 0 + onnx_model.doc_string = '' + set_model_props(onnx_model, {}) + + del onnx_model.opset_import[:] # pylint: disable=E1101 + for dom, value in opsets.items(): + op_set = onnx_model.opset_import.add() + op_set.domain = dom + op_set.version = value + + return onnx_model + + onnx_model = create_model() + + oinf = OnnxInference(onnx_model) + xs = [ + array([[[1.7640524, 0.4001572, 0.978738, 2.2408931, + 1.867558], + [-0.9772779, 0.95008844, -0.1513572, -0.10321885, + 0.41059852], + [0.14404356, 1.4542735, 0.7610377, 0.12167501, + 0.44386324], + [0.33367434, 1.4940791, -0.20515826, 0.3130677, + -0.85409576]], + + [[-2.5529897, 0.6536186, 0.8644362, -0.742165, + 2.2697546], + [-1.4543657, 0.04575852, -0.18718386, 1.5327792, + 1.4693588], + [0.15494743, 0.37816253, -0.88778573, -1.9807965, + -0.34791216], + [0.15634897, 1.2302907, 1.2023798, -0.3873268, + -0.30230275]], + + [[-1.048553, -1.420018, -1.7062702, 1.9507754, + -0.5096522], + [-0.4380743, -1.2527953, 0.7774904, -1.6138978, + -0.21274029], + [-0.89546657, 0.3869025, -0.51080513, -1.1806322, + -0.02818223], + [0.42833188, 0.06651722, 0.3024719, -0.6343221, + -0.36274117]]], dtype=float32), + ] + ys = [ + array([[[1.7640524, 0.4001572, 0.978738, 2.2408931, 1.867558], + [0.9772779, 0.95008844, 0.1513572, 0.10321885, 0.41059852], + [0.14404356, 1.4542735, 0.7610377, 0.12167501, 0.44386324], + [0.33367434, 1.4940791, 0.20515826, 0.3130677, 0.85409576]], + + [[2.5529897, 0.6536186, 0.8644362, 0.742165, 2.2697546], + [1.4543657, 0.04575852, 0.18718386, 1.5327792, 1.4693588], + [0.15494743, 0.37816253, 0.88778573, 1.9807965, 0.34791216], + [0.15634897, 1.2302907, 1.2023798, 0.3873268, 0.30230275]], + + [[1.048553, 1.420018, 1.7062702, 1.9507754, 0.5096522], + [0.4380743, 1.2527953, 0.7774904, 1.6138978, 0.21274029], + [0.89546657, 0.3869025, 0.51080513, 1.1806322, 0.02818223], + [0.42833188, 0.06651722, 0.3024719, 0.6343221, 0.36274117]]], + dtype=float32), + ] + feeds = {n: x for n, x in zip(oinf.input_names, xs)} + got = oinf.run(feeds) + goty = [got[k] for k in oinf.output_names] + for y, gy in zip(ys, goty): + self.assertEqualArray(y, gy) if __name__ == "__main__": diff --git a/mlprodict/testing/onnx_backend.py b/mlprodict/testing/onnx_backend.py index 2517838f6..e0d60cefe 100644 --- a/mlprodict/testing/onnx_backend.py +++ b/mlprodict/testing/onnx_backend.py @@ -3,6 +3,7 @@ @brief Tests with onnx backend. """ import os +import textwrap from numpy.testing import assert_almost_equal import onnx from onnx.numpy_helper import to_array @@ -126,6 +127,47 @@ def run(self, load_fct, run_fct, index=None, decimal=5): "Output %d of test %d in folder %r failed." % ( i, index, self.folder)) from ex + def to_python(self): + """ + Returns a python code equivalent to the ONNX test. + + :return: code + """ + from ..onnx_tools.onnx_export import export2onnx + rows = [] + code = export2onnx(self.onnx_model) + lines = code.split('\n') + lines = [line for line in lines + if not line.strip().startswith('print') and + not line.strip().startswith('# ')] + rows.append(textwrap.dedent("\n".join(lines))) + rows.append("oinf = OnnxInference(onnx_model)") + for test in self.tests: + rows.append("xs = [") + for inp in test['inputs']: + rows.append(textwrap.indent(repr(inp) + ',', + ' ' * 2)) + rows.append("]") + rows.append("ys = [") + for out in test['outputs']: + rows.append(textwrap.indent(repr(out) + ',', + ' ' * 2)) + rows.append("]") + rows.append("feeds = {n: x for n, x in zip(oinf.input_names, xs)}") + rows.append("got = oinf.run(feeds)") + rows.append("goty = [got[k] for k in oinf.output_names]") + rows.append("for y, gy in zip(ys, goty):") + rows.append(" self.assertEqualArray(y, gy)") + rows.append("") + code = "\n".join(rows) + final = "\n".join(["def %s(self):" % self.name, + textwrap.indent(code, ' ')]) + try: + from pyquickhelper.pycode.code_helper import remove_extra_spaces_and_pep8 + except ImportError: + return final + return remove_extra_spaces_and_pep8(final) + def enumerate_onnx_tests(series, fct_filter=None): """ From d74780cd04f8a0677cd634bf62ca4d4b6d7578f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Sat, 5 Mar 2022 23:38:04 +0100 Subject: [PATCH 2/2] Update test_onnx_backend.py --- _unittests/ut_testing/test_onnx_backend.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/_unittests/ut_testing/test_onnx_backend.py b/_unittests/ut_testing/test_onnx_backend.py index e4970a46e..b7d9475e6 100644 --- a/_unittests/ut_testing/test_onnx_backend.py +++ b/_unittests/ut_testing/test_onnx_backend.py @@ -1,12 +1,10 @@ """ -@brief test log(time=10s) +@brief test log(time=40s) """ import unittest -import numpy from numpy import array, float32 -from onnx import numpy_helper, TensorProto from onnx.helper import ( - make_model, make_node, set_model_props, make_tensor, make_graph, + make_model, make_node, set_model_props, make_graph, make_tensor_value_info) from pyquickhelper.pycode import ExtTestCase from mlprodict.testing.onnx_backend import enumerate_onnx_tests @@ -102,7 +100,6 @@ def create_model(): outputs = [] opsets = {'': 9} - target_opset = 9 # subgraphs value = make_tensor_value_info('x', 1, [3, 4, 5]) inputs.append(value) @@ -117,8 +114,7 @@ def create_model(): domain='') nodes.append(node) - graph = make_graph(nodes, 'test_abs', inputs, - outputs, initializers) + graph = make_graph(nodes, 'test_abs', inputs, outputs, initializers) onnx_model = make_model(graph) onnx_model.ir_version = 3