From 66d711919da5386123cd178852fba6e499aa31d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Sat, 5 Feb 2022 11:17:52 +0100 Subject: [PATCH 01/10] Moves OnnxMicroRuntime to onnxrt --- _doc/sphinxdoc/source/api/onnxrt.rst | 5 +- _doc/sphinxdoc/source/api/tools.rst | 7 - .../test_onnx_micro_runtime.py | 242 +++++------ mlprodict/onnxrt/__init__.py | 2 + .../{tools => onnxrt}/onnx_micro_runtime.py | 386 +++++++++--------- mlprodict/testing/einsum/einsum_fct.py | 2 +- 6 files changed, 321 insertions(+), 323 deletions(-) rename _unittests/{ut_tools => ut_onnxrt}/test_onnx_micro_runtime.py (96%) rename mlprodict/{tools => onnxrt}/onnx_micro_runtime.py (97%) diff --git a/_doc/sphinxdoc/source/api/onnxrt.rst b/_doc/sphinxdoc/source/api/onnxrt.rst index 9297abbb8..d76aa1a66 100644 --- a/_doc/sphinxdoc/source/api/onnxrt.rst +++ b/_doc/sphinxdoc/source/api/onnxrt.rst @@ -18,7 +18,10 @@ implementated in :epkg:`Python`. The :epkg:`ONNX` model relies on the following operators :ref:`l-onnx-runtime-operators`. .. autosignature:: mlprodict.onnxrt.onnx_inference.OnnxInference - :members: + :members: run, shape_inference, check_model, run2onnx, get_profiling + +.. autosignature:: mlprodict.onnxrt.onnx_micro_inference.OnnxMicroRuntime + :members: run Python to ONNX ++++++++++++++ diff --git a/_doc/sphinxdoc/source/api/tools.rst b/_doc/sphinxdoc/source/api/tools.rst index ac51a43a6..2fa3d5749 100644 --- a/_doc/sphinxdoc/source/api/tools.rst +++ b/_doc/sphinxdoc/source/api/tools.rst @@ -83,13 +83,6 @@ Serialization .. autosignature:: mlprodict.onnx_tools.onnx2py_helper.to_bytes -Runtime -======= - -.. autosignature:: mlprodict.onnxrt.onnx_inference.OnnxInference - -.. autosignature:: mlprodict.tools.onnx_micro_runtime.OnnxMicroRuntime - Validation ++++++++++ diff --git a/_unittests/ut_tools/test_onnx_micro_runtime.py b/_unittests/ut_onnxrt/test_onnx_micro_runtime.py similarity index 96% rename from _unittests/ut_tools/test_onnx_micro_runtime.py rename to _unittests/ut_onnxrt/test_onnx_micro_runtime.py index 8fa35daa3..d82defadb 100644 --- a/_unittests/ut_tools/test_onnx_micro_runtime.py +++ b/_unittests/ut_onnxrt/test_onnx_micro_runtime.py @@ -1,121 +1,121 @@ -""" -@brief test log(time=3s) -""" -import unittest -import numpy -from pyquickhelper.pycode import ExtTestCase -from skl2onnx.algebra.onnx_ops import ( # pylint: disable=E0611 - OnnxAdd, OnnxTranspose, OnnxShape, OnnxPow, OnnxMatMul, OnnxGemm, - OnnxSqueeze, OnnxUnsqueeze) -from mlprodict.tools.onnx_micro_runtime import OnnxMicroRuntime - - -class TestOnnxMicroRuntime(ExtTestCase): - - opset = 15 # opset=13, 14, ... - - def test_onnx_micro_runtime(self): - opset = TestOnnxMicroRuntime.opset - dtype = numpy.float32 - x = numpy.array([1, 2, 4, 5, 5, 4]).astype( - numpy.float32).reshape((3, 2)) - cop = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=opset) - cop4 = OnnxAdd(cop, numpy.array([2], dtype=dtype), op_version=opset, - output_names=['Y']) - model_def = cop4.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - out = rt.run({'X': x}) - self.assertIn('X', out) - self.assertIn('Y', out) - self.assertIn('Ad_Addcst', out) - self.assertEqual(len(out), 5) - - def test_onnx_micro_runtime_exc1(self): - self.assertRaise(lambda: OnnxMicroRuntime(None), TypeError) - - def test_onnx_micro_runtime_exc2(self): - opset = TestOnnxMicroRuntime.opset - dtype = numpy.float32 - x = numpy.array([1, 2, 4, 5, 5, 4]).astype( - numpy.float32).reshape((3, 2)) - cop = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=opset) - cop4 = OnnxPow(cop, numpy.array([2], dtype=dtype), op_version=opset, - output_names=['Y']) - model_def = cop4.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - self.assertRaise(lambda: rt.run({'X': x}), NotImplementedError) - self.assertRaise(lambda: rt.run(x), TypeError) - - def test_onnx_micro_runtime_shape(self): - opset = TestOnnxMicroRuntime.opset - x = numpy.array([1, 2, 4, 5, 5, 4]).astype( - numpy.float32).reshape((3, 2)) - cop = OnnxShape('X', op_version=opset, output_names=['Y']) - model_def = cop.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - out = rt.run({'X': x}) - self.assertEqual(numpy.array(x.shape, dtype=numpy.int64), out['Y']) - - def test_onnx_micro_runtime_transpose(self): - opset = TestOnnxMicroRuntime.opset - x = numpy.array([1, 2, 4, 5, 5, 4]).astype( - numpy.float32).reshape((3, 2)) - cop = OnnxTranspose('X', perm=[1, 0], op_version=opset, - output_names=['Y']) - model_def = cop.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - out = rt.run({'X': x}) - self.assertEqual(x.T, out['Y']) - - def test_onnx_micro_runtime_matmul(self): - opset = TestOnnxMicroRuntime.opset - x = numpy.array([1, 2, 4, 5]).astype( - numpy.float32).reshape((2, 2)) - cop = OnnxMatMul('X', 'X', op_version=opset, - output_names=['Y']) - model_def = cop.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - out = rt.run({'X': x}) - self.assertEqual(numpy.matmul(x, x), out['Y']) - - def test_onnx_micro_runtime_squeeze(self): - opset = TestOnnxMicroRuntime.opset - x = numpy.array([1, 2, 4, 5]).astype( - numpy.float32).reshape((2, 2, 1)) - cop = OnnxSqueeze('X', numpy.array([2], dtype=numpy.int64), - op_version=opset, output_names=['Y']) - model_def = cop.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - out = rt.run({'X': x}) - self.assertEqual(numpy.squeeze(x), out['Y']) - - def test_onnx_micro_runtime_unsqueeze(self): - opset = TestOnnxMicroRuntime.opset - x = numpy.array([1, 2, 4, 5]).astype( - numpy.float32).reshape((2, 2)) - cop = OnnxUnsqueeze('X', numpy.array([2], dtype=numpy.int64), - op_version=opset, output_names=['Y']) - model_def = cop.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - out = rt.run({'X': x}) - self.assertEqual(x.reshape((2, 2, 1)), out['Y']) - - def test_onnx_micro_runtime_gemm(self): - opset = TestOnnxMicroRuntime.opset - x = numpy.array([1, 2, 4, 5]).astype( - numpy.float32).reshape((2, 2)) - for ta in [0, 1]: - for tb in [0, 1]: - cop = OnnxGemm( - 'X', 'X', 'X', op_version=opset, alpha=1., beta=1., - output_names=['Y'], transA=ta, transB=tb) - model_def = cop.to_onnx({'X': x}, target_opset=opset) - rt = OnnxMicroRuntime(model_def) - out = rt.run({'X': x}) - xa = x.T if ta else x - xb = x.T if tb else x - self.assertEqual(numpy.matmul(xa, xb) + x, out['Y']) - - -if __name__ == "__main__": - unittest.main() +""" +@brief test log(time=3s) +""" +import unittest +import numpy +from pyquickhelper.pycode import ExtTestCase +from skl2onnx.algebra.onnx_ops import ( # pylint: disable=E0611 + OnnxAdd, OnnxTranspose, OnnxShape, OnnxPow, OnnxMatMul, OnnxGemm, + OnnxSqueeze, OnnxUnsqueeze) +from mlprodict.onnxrt.onnx_micro_runtime import OnnxMicroRuntime + + +class TestOnnxMicroRuntime(ExtTestCase): + + opset = 15 # opset=13, 14, ... + + def test_onnx_micro_runtime(self): + opset = TestOnnxMicroRuntime.opset + dtype = numpy.float32 + x = numpy.array([1, 2, 4, 5, 5, 4]).astype( + numpy.float32).reshape((3, 2)) + cop = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=opset) + cop4 = OnnxAdd(cop, numpy.array([2], dtype=dtype), op_version=opset, + output_names=['Y']) + model_def = cop4.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + out = rt.run({'X': x}) + self.assertIn('X', out) + self.assertIn('Y', out) + self.assertIn('Ad_Addcst', out) + self.assertEqual(len(out), 5) + + def test_onnx_micro_runtime_exc1(self): + self.assertRaise(lambda: OnnxMicroRuntime(None), TypeError) + + def test_onnx_micro_runtime_exc2(self): + opset = TestOnnxMicroRuntime.opset + dtype = numpy.float32 + x = numpy.array([1, 2, 4, 5, 5, 4]).astype( + numpy.float32).reshape((3, 2)) + cop = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=opset) + cop4 = OnnxPow(cop, numpy.array([2], dtype=dtype), op_version=opset, + output_names=['Y']) + model_def = cop4.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + self.assertRaise(lambda: rt.run({'X': x}), NotImplementedError) + self.assertRaise(lambda: rt.run(x), TypeError) + + def test_onnx_micro_runtime_shape(self): + opset = TestOnnxMicroRuntime.opset + x = numpy.array([1, 2, 4, 5, 5, 4]).astype( + numpy.float32).reshape((3, 2)) + cop = OnnxShape('X', op_version=opset, output_names=['Y']) + model_def = cop.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + out = rt.run({'X': x}) + self.assertEqual(numpy.array(x.shape, dtype=numpy.int64), out['Y']) + + def test_onnx_micro_runtime_transpose(self): + opset = TestOnnxMicroRuntime.opset + x = numpy.array([1, 2, 4, 5, 5, 4]).astype( + numpy.float32).reshape((3, 2)) + cop = OnnxTranspose('X', perm=[1, 0], op_version=opset, + output_names=['Y']) + model_def = cop.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + out = rt.run({'X': x}) + self.assertEqual(x.T, out['Y']) + + def test_onnx_micro_runtime_matmul(self): + opset = TestOnnxMicroRuntime.opset + x = numpy.array([1, 2, 4, 5]).astype( + numpy.float32).reshape((2, 2)) + cop = OnnxMatMul('X', 'X', op_version=opset, + output_names=['Y']) + model_def = cop.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + out = rt.run({'X': x}) + self.assertEqual(numpy.matmul(x, x), out['Y']) + + def test_onnx_micro_runtime_squeeze(self): + opset = TestOnnxMicroRuntime.opset + x = numpy.array([1, 2, 4, 5]).astype( + numpy.float32).reshape((2, 2, 1)) + cop = OnnxSqueeze('X', numpy.array([2], dtype=numpy.int64), + op_version=opset, output_names=['Y']) + model_def = cop.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + out = rt.run({'X': x}) + self.assertEqual(numpy.squeeze(x), out['Y']) + + def test_onnx_micro_runtime_unsqueeze(self): + opset = TestOnnxMicroRuntime.opset + x = numpy.array([1, 2, 4, 5]).astype( + numpy.float32).reshape((2, 2)) + cop = OnnxUnsqueeze('X', numpy.array([2], dtype=numpy.int64), + op_version=opset, output_names=['Y']) + model_def = cop.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + out = rt.run({'X': x}) + self.assertEqual(x.reshape((2, 2, 1)), out['Y']) + + def test_onnx_micro_runtime_gemm(self): + opset = TestOnnxMicroRuntime.opset + x = numpy.array([1, 2, 4, 5]).astype( + numpy.float32).reshape((2, 2)) + for ta in [0, 1]: + for tb in [0, 1]: + cop = OnnxGemm( + 'X', 'X', 'X', op_version=opset, alpha=1., beta=1., + output_names=['Y'], transA=ta, transB=tb) + model_def = cop.to_onnx({'X': x}, target_opset=opset) + rt = OnnxMicroRuntime(model_def) + out = rt.run({'X': x}) + xa = x.T if ta else x + xb = x.T if tb else x + self.assertEqual(numpy.matmul(xa, xb) + x, out['Y']) + + +if __name__ == "__main__": + unittest.main() diff --git a/mlprodict/onnxrt/__init__.py b/mlprodict/onnxrt/__init__.py index b3b53d01c..1ac63bc46 100644 --- a/mlprodict/onnxrt/__init__.py +++ b/mlprodict/onnxrt/__init__.py @@ -4,3 +4,5 @@ @brief Shortcut to *onnxrt*. """ from .onnx_inference import OnnxInference +from .onnx_micro_runtime import OnnxMicroRuntime + diff --git a/mlprodict/tools/onnx_micro_runtime.py b/mlprodict/onnxrt/onnx_micro_runtime.py similarity index 97% rename from mlprodict/tools/onnx_micro_runtime.py rename to mlprodict/onnxrt/onnx_micro_runtime.py index 6a2217cda..250f881af 100644 --- a/mlprodict/tools/onnx_micro_runtime.py +++ b/mlprodict/onnxrt/onnx_micro_runtime.py @@ -1,193 +1,193 @@ -""" -@file -@brief Micro runtime for ONNX. - -.. versionadded:: 0.6 -""" -import numpy -from ..onnx_tools.onnx2py_helper import _var_as_dict - - -class OnnxMicroRuntime: - """ - Implements a micro runtime for ONNX graphs. - It does not implements all the operator types. - - :param model_onnx: ONNX model - """ - - def __init__(self, model_onnx): - if not hasattr(model_onnx, 'graph'): - raise TypeError( - "model_onnx is not an ONNX graph but %r." % type(model_onnx)) - self.model_onnx = model_onnx - - def run(self, inputs): - """ - Computes the outputs of the graph. - - :param inputs: dictionary - :return: all intermediates results and output as a dictionary - """ - if not isinstance(inputs, dict): - raise TypeError( - "inputs must be a dictionary not %r." % type(inputs)) - results = inputs.copy() - - for init in self.model_onnx.graph.initializer: - name = init.name - mat = _var_as_dict(init)['value'] - results[name] = mat - - for node in self.model_onnx.graph.node: - op_type = node.op_type - inp = [results[n] for n in node.input] - meth_name = "_op_%s" % op_type.lower() - if not hasattr(self, meth_name): - raise NotImplementedError( - "OnnxMicroRuntime does not implement operator %r." % op_type) - kwargs = {} - for at in node.attribute: - var = _var_as_dict(at) - kwargs[at.name] = var['value'] - out = getattr(self, meth_name)(*inp, **kwargs) - for n, o in zip(node.output, out): - results[n] = o - - return results - - ######################## - # Runtime for operators - ######################## - - def _op_add(self, x, y): - "Runtime for operator :epkg:`Op:Add`." - return (x + y, ) - - def _op_concat(self, *args, axis=None): - "Runtime for operator :epkg:`Op:Concat`." - def _preprocess(a, axis): - if axis >= len(a.shape): - new_shape = a.shape + (1, ) * (axis + 1 - len(a.shape)) - return a.reshape(new_shape) - return a - - targs = tuple(_preprocess(a, axis) for a in args) - return (numpy.concatenate(targs, axis), ) - - def _op_gemm(self, a, b, c=None, alpha=None, beta=None, - transA=False, transB=False): - "Runtime for operator :epkg:`Op:Gemm`." - - def _gemm00(a, b, c, alpha, beta): - o = numpy.dot(a, b) * alpha - if beta != 0: - o += c * beta - return o - - def _gemm01(a, b, c, alpha, beta): - o = numpy.dot(a, b.T) * alpha - if beta != 0: - o += c * beta - return o - - def _gemm10(a, b, c, alpha, beta): - o = numpy.dot(a.T, b) * alpha - if beta != 0: - o += c * beta - return o - - def _gemm11(a, b, c, alpha, beta): - o = numpy.dot(a.T, b.T) * alpha - if beta != 0: - o += c * beta - return o - - if not isinstance(transA, (int, bool, numpy.int64)): - raise TypeError( # pragma: no cover - "Unexpected type for transA: %r." % type(transA)) - if not isinstance(transB, (int, bool, numpy.int64)): - raise TypeError( # pragma: no cover - "Unexpected type for transA: %r." % type(transB)) - if transA: - fct = _gemm11 if transB else _gemm10 - else: - fct = _gemm01 if transB else _gemm00 - return (fct(a, b, c, alpha=alpha, beta=beta), ) - - def _op_gather(self, x, indices, axis=None): - "Runtime for operator :epkg:`Op:Gather`." - if not x.flags['C_CONTIGUOUS']: - x = numpy.ascontiguousarray(x) - if not indices.flags['C_CONTIGUOUS']: - indices = indices.ascontiguousarray() - return (numpy.take(x, indices, axis=axis), ) - - def _op_identity(self, x): - "Runtime for operator :epkg:`Op:Identity`." - return (x, ) - - def _op_matmul(self, x, y): - "Runtime for operator :epkg:`Op:MatMul`." - return (numpy.matmul(x, y), ) - - def _op_max(self, *inps): - "Runtime for operator :epkg:`Op:Max`." - return (numpy.maximum(*inps), ) - - def _op_mul(self, x, y): - "Runtime for operator :epkg:`Op:Mul`." - return (x * y, ) - - def _op_reduceprod(self, data, axes=None, keepdims=None): - "Runtime for operator :epkg:`Op:ReduceProd`." - if axes is not None and not isinstance(axes, int): - if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: - axes = int(axes) - else: - axes = tuple(axes) if len(axes) > 0 else None - return (numpy.prod(data, axis=axes, - keepdims=keepdims, - dtype=data.dtype), ) - - def _op_reducesum(self, data, axes, keepdims=None, - noop_with_empty_axes=None): - "Runtime for operator :epkg:`Op:ReduceSum`." - if axes is None and noop_with_empty_axes: - return (data, ) - if axes is not None and not isinstance(axes, int): - if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: - axes = int(axes) - else: - axes = tuple(axes) if len(axes) > 0 else None - return (numpy.sum(data, axis=axes, - keepdims=keepdims, - dtype=data.dtype), ) - - def _op_reshape(self, x, shape): - "Runtime for operator :epkg:`Op:Reshape`." - return (x.reshape(shape), ) - - def _op_shape(self, x): - "Runtime for operator :epkg:`Op:Shape`." - return (numpy.array(list(x.shape), dtype=numpy.int64), ) - - def _op_squeeze(self, x, axes=None): - "Runtime for operator :epkg:`Op:Squeeze`." - if axes is None: - return (x, ) - if hasattr(axes, '__iter__'): - return (numpy.squeeze(x, axis=tuple(axes)), ) - return (numpy.squeeze(x, axis=axes), ) - - def _op_transpose(self, x, perm=None): - "Runtime for operator :epkg:`Op:Transpose`." - return (numpy.transpose(x, perm), ) - - def _op_unsqueeze(self, x, axes=None): - "Runtime for operator :epkg:`Op:Unsqueeze`." - if axes is None: - return (x, ) - if hasattr(axes, '__iter__'): - return (numpy.expand_dims(x, axis=tuple(axes)), ) - return (numpy.expand_dims(x, axis=axes), ) +""" +@file +@brief Micro runtime for ONNX. + +.. versionadded:: 0.6 +""" +import numpy +from ..onnx_tools.onnx2py_helper import _var_as_dict + + +class OnnxMicroRuntime: + """ + Implements a micro runtime for ONNX graphs. + It does not implements all the operator types. + + :param model_onnx: ONNX model + """ + + def __init__(self, model_onnx): + if not hasattr(model_onnx, 'graph'): + raise TypeError( + "model_onnx is not an ONNX graph but %r." % type(model_onnx)) + self.model_onnx = model_onnx + + def run(self, inputs): + """ + Computes the outputs of the graph. + + :param inputs: dictionary + :return: all intermediates results and output as a dictionary + """ + if not isinstance(inputs, dict): + raise TypeError( + "inputs must be a dictionary not %r." % type(inputs)) + results = inputs.copy() + + for init in self.model_onnx.graph.initializer: + name = init.name + mat = _var_as_dict(init)['value'] + results[name] = mat + + for node in self.model_onnx.graph.node: + op_type = node.op_type + inp = [results[n] for n in node.input] + meth_name = "_op_%s" % op_type.lower() + if not hasattr(self, meth_name): + raise NotImplementedError( + "OnnxMicroRuntime does not implement operator %r." % op_type) + kwargs = {} + for at in node.attribute: + var = _var_as_dict(at) + kwargs[at.name] = var['value'] + out = getattr(self, meth_name)(*inp, **kwargs) + for n, o in zip(node.output, out): + results[n] = o + + return results + + ######################## + # Runtime for operators + ######################## + + def _op_add(self, x, y): + "Runtime for operator :epkg:`Op:Add`." + return (x + y, ) + + def _op_concat(self, *args, axis=None): + "Runtime for operator :epkg:`Op:Concat`." + def _preprocess(a, axis): + if axis >= len(a.shape): + new_shape = a.shape + (1, ) * (axis + 1 - len(a.shape)) + return a.reshape(new_shape) + return a + + targs = tuple(_preprocess(a, axis) for a in args) + return (numpy.concatenate(targs, axis), ) + + def _op_gemm(self, a, b, c=None, alpha=None, beta=None, + transA=False, transB=False): + "Runtime for operator :epkg:`Op:Gemm`." + + def _gemm00(a, b, c, alpha, beta): + o = numpy.dot(a, b) * alpha + if beta != 0: + o += c * beta + return o + + def _gemm01(a, b, c, alpha, beta): + o = numpy.dot(a, b.T) * alpha + if beta != 0: + o += c * beta + return o + + def _gemm10(a, b, c, alpha, beta): + o = numpy.dot(a.T, b) * alpha + if beta != 0: + o += c * beta + return o + + def _gemm11(a, b, c, alpha, beta): + o = numpy.dot(a.T, b.T) * alpha + if beta != 0: + o += c * beta + return o + + if not isinstance(transA, (int, bool, numpy.int64)): + raise TypeError( # pragma: no cover + "Unexpected type for transA: %r." % type(transA)) + if not isinstance(transB, (int, bool, numpy.int64)): + raise TypeError( # pragma: no cover + "Unexpected type for transA: %r." % type(transB)) + if transA: + fct = _gemm11 if transB else _gemm10 + else: + fct = _gemm01 if transB else _gemm00 + return (fct(a, b, c, alpha=alpha, beta=beta), ) + + def _op_gather(self, x, indices, axis=None): + "Runtime for operator :epkg:`Op:Gather`." + if not x.flags['C_CONTIGUOUS']: + x = numpy.ascontiguousarray(x) + if not indices.flags['C_CONTIGUOUS']: + indices = indices.ascontiguousarray() + return (numpy.take(x, indices, axis=axis), ) + + def _op_identity(self, x): + "Runtime for operator :epkg:`Op:Identity`." + return (x, ) + + def _op_matmul(self, x, y): + "Runtime for operator :epkg:`Op:MatMul`." + return (numpy.matmul(x, y), ) + + def _op_max(self, *inps): + "Runtime for operator :epkg:`Op:Max`." + return (numpy.maximum(*inps), ) + + def _op_mul(self, x, y): + "Runtime for operator :epkg:`Op:Mul`." + return (x * y, ) + + def _op_reduceprod(self, data, axes=None, keepdims=None): + "Runtime for operator :epkg:`Op:ReduceProd`." + if axes is not None and not isinstance(axes, int): + if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: + axes = int(axes) + else: + axes = tuple(axes) if len(axes) > 0 else None + return (numpy.prod(data, axis=axes, + keepdims=keepdims, + dtype=data.dtype), ) + + def _op_reducesum(self, data, axes, keepdims=None, + noop_with_empty_axes=None): + "Runtime for operator :epkg:`Op:ReduceSum`." + if axes is None and noop_with_empty_axes: + return (data, ) + if axes is not None and not isinstance(axes, int): + if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: + axes = int(axes) + else: + axes = tuple(axes) if len(axes) > 0 else None + return (numpy.sum(data, axis=axes, + keepdims=keepdims, + dtype=data.dtype), ) + + def _op_reshape(self, x, shape): + "Runtime for operator :epkg:`Op:Reshape`." + return (x.reshape(shape), ) + + def _op_shape(self, x): + "Runtime for operator :epkg:`Op:Shape`." + return (numpy.array(list(x.shape), dtype=numpy.int64), ) + + def _op_squeeze(self, x, axes=None): + "Runtime for operator :epkg:`Op:Squeeze`." + if axes is None: + return (x, ) + if hasattr(axes, '__iter__'): + return (numpy.squeeze(x, axis=tuple(axes)), ) + return (numpy.squeeze(x, axis=axes), ) + + def _op_transpose(self, x, perm=None): + "Runtime for operator :epkg:`Op:Transpose`." + return (numpy.transpose(x, perm), ) + + def _op_unsqueeze(self, x, axes=None): + "Runtime for operator :epkg:`Op:Unsqueeze`." + if axes is None: + return (x, ) + if hasattr(axes, '__iter__'): + return (numpy.expand_dims(x, axis=tuple(axes)), ) + return (numpy.expand_dims(x, axis=axes), ) diff --git a/mlprodict/testing/einsum/einsum_fct.py b/mlprodict/testing/einsum/einsum_fct.py index 1830146de..589698cf7 100644 --- a/mlprodict/testing/einsum/einsum_fct.py +++ b/mlprodict/testing/einsum/einsum_fct.py @@ -10,7 +10,7 @@ from onnx import helper from skl2onnx.common.data_types import FloatTensorType from ...onnx_tools.onnx2py_helper import guess_proto_dtype -from ...tools.onnx_micro_runtime import OnnxMicroRuntime +from ...onnxrt.onnx_micro_runtime import OnnxMicroRuntime from ...tools.asv_options_helper import ( get_opset_number_from_onnx, get_ir_version_from_onnx) from .einsum_impl import decompose_einsum_equation, apply_einsum_sequence From 9b6ca446cf73abbc64a99a562b8d2df18efd7487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Sat, 5 Feb 2022 11:48:09 +0100 Subject: [PATCH 02/10] first commit --- _unittests/ut_onnxrt/test_shape_inference.py | 35 ++++ mlprodict/onnxrt/__init__.py | 2 +- mlprodict/onnxrt/onnx_shape_inference.py | 193 +++++++++++++++++++ mlprodict/onnxrt/ops_shape/__init__.py | 9 + mlprodict/onnxrt/ops_shape/_element_wise.py | 33 ++++ mlprodict/onnxrt/ops_shape/shape_result.py | 40 ++++ 6 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 _unittests/ut_onnxrt/test_shape_inference.py create mode 100644 mlprodict/onnxrt/onnx_shape_inference.py create mode 100644 mlprodict/onnxrt/ops_shape/__init__.py create mode 100644 mlprodict/onnxrt/ops_shape/_element_wise.py create mode 100644 mlprodict/onnxrt/ops_shape/shape_result.py diff --git a/_unittests/ut_onnxrt/test_shape_inference.py b/_unittests/ut_onnxrt/test_shape_inference.py new file mode 100644 index 000000000..999fef77a --- /dev/null +++ b/_unittests/ut_onnxrt/test_shape_inference.py @@ -0,0 +1,35 @@ +""" +@brief test log(time=3s) +""" +import unittest +import numpy +from pyquickhelper.pycode import ExtTestCase +from skl2onnx.algebra.onnx_ops import ( # pylint: disable=E0611 + OnnxAdd, OnnxTranspose, OnnxShape, OnnxPow, OnnxMatMul, OnnxGemm, + OnnxSqueeze, OnnxUnsqueeze) +from mlprodict.onnxrt import OnnxShapeInference + + +class TestOnnxShapeInference(ExtTestCase): + + opsets = (10, onnx. + + def test_onnx_micro_runtime(self): + opset = OnnxShapeInference.opset + dtype = numpy.float32 + x = numpy.array([1, 2, 4, 5, 5, 4]).astype( + numpy.float32).reshape((3, 2)) + cop = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=opset) + cop4 = OnnxAdd(cop, numpy.array([2], dtype=dtype), op_version=opset, + output_names=['Y']) + model_def = cop4.to_onnx({'X': x}, target_opset=opset) + rt = OnnxShapeInference(model_def) + out = rt.run({'X': x}) + self.assertIn('X', out) + self.assertIn('Y', out) + self.assertIn('Ad_Addcst', out) + self.assertEqual(len(out), 5) + + +if __name__ == "__main__": + unittest.main() diff --git a/mlprodict/onnxrt/__init__.py b/mlprodict/onnxrt/__init__.py index 1ac63bc46..9611a88d6 100644 --- a/mlprodict/onnxrt/__init__.py +++ b/mlprodict/onnxrt/__init__.py @@ -5,4 +5,4 @@ """ from .onnx_inference import OnnxInference from .onnx_micro_runtime import OnnxMicroRuntime - +from .onnx_shape_inference import OnnxShapeInference diff --git a/mlprodict/onnxrt/onnx_shape_inference.py b/mlprodict/onnxrt/onnx_shape_inference.py new file mode 100644 index 000000000..c2c6ee46e --- /dev/null +++ b/mlprodict/onnxrt/onnx_shape_inference.py @@ -0,0 +1,193 @@ +""" +@file +@brief Runtime to infer shapes. + +.. versionadded:: 0.9 +""" +import numpy +from ..onnx_tools.onnx2py_helper import _var_as_dict + + +class OnnxShapeInference: + """ + Implements a micro runtime for ONNX graphs. + It does not implements all the operator types. + + :param model_onnx: ONNX model + """ + + def __init__(self, model_onnx): + if not hasattr(model_onnx, 'graph'): + raise TypeError( + "model_onnx is not an ONNX graph but %r." % type(model_onnx)) + self.model_onnx = model_onnx + + def run(self, inputs): + """ + Computes the outputs of the graph. + + :param inputs: dictionary + :return: all intermediates results and output as a dictionary + """ + if not isinstance(inputs, dict): + raise TypeError( + "inputs must be a dictionary not %r." % type(inputs)) + results = inputs.copy() + + for init in self.model_onnx.graph.initializer: + name = init.name + mat = _var_as_dict(init)['value'] + results[name] = mat + + for node in self.model_onnx.graph.node: + op_type = node.op_type + inp = [results[n] for n in node.input] + meth_name = "_op_%s" % op_type.lower() + if not hasattr(self, meth_name): + raise NotImplementedError( + "OnnxMicroRuntime does not implement operator %r." % op_type) + kwargs = {} + for at in node.attribute: + var = _var_as_dict(at) + kwargs[at.name] = var['value'] + out = getattr(self, meth_name)(*inp, **kwargs) + for n, o in zip(node.output, out): + results[n] = o + + return results + + ######################## + # Runtime for operators + ######################## + + def _op_add(self, x, y): + "Runtime for operator :epkg:`Op:Add`." + return (x + y, ) + + def _op_concat(self, *args, axis=None): + "Runtime for operator :epkg:`Op:Concat`." + def _preprocess(a, axis): + if axis >= len(a.shape): + new_shape = a.shape + (1, ) * (axis + 1 - len(a.shape)) + return a.reshape(new_shape) + return a + + targs = tuple(_preprocess(a, axis) for a in args) + return (numpy.concatenate(targs, axis), ) + + def _op_gemm(self, a, b, c=None, alpha=None, beta=None, + transA=False, transB=False): + "Runtime for operator :epkg:`Op:Gemm`." + + def _gemm00(a, b, c, alpha, beta): + o = numpy.dot(a, b) * alpha + if beta != 0: + o += c * beta + return o + + def _gemm01(a, b, c, alpha, beta): + o = numpy.dot(a, b.T) * alpha + if beta != 0: + o += c * beta + return o + + def _gemm10(a, b, c, alpha, beta): + o = numpy.dot(a.T, b) * alpha + if beta != 0: + o += c * beta + return o + + def _gemm11(a, b, c, alpha, beta): + o = numpy.dot(a.T, b.T) * alpha + if beta != 0: + o += c * beta + return o + + if not isinstance(transA, (int, bool, numpy.int64)): + raise TypeError( # pragma: no cover + "Unexpected type for transA: %r." % type(transA)) + if not isinstance(transB, (int, bool, numpy.int64)): + raise TypeError( # pragma: no cover + "Unexpected type for transA: %r." % type(transB)) + if transA: + fct = _gemm11 if transB else _gemm10 + else: + fct = _gemm01 if transB else _gemm00 + return (fct(a, b, c, alpha=alpha, beta=beta), ) + + def _op_gather(self, x, indices, axis=None): + "Runtime for operator :epkg:`Op:Gather`." + if not x.flags['C_CONTIGUOUS']: + x = numpy.ascontiguousarray(x) + if not indices.flags['C_CONTIGUOUS']: + indices = indices.ascontiguousarray() + return (numpy.take(x, indices, axis=axis), ) + + def _op_identity(self, x): + "Runtime for operator :epkg:`Op:Identity`." + return (x, ) + + def _op_matmul(self, x, y): + "Runtime for operator :epkg:`Op:MatMul`." + return (numpy.matmul(x, y), ) + + def _op_max(self, *inps): + "Runtime for operator :epkg:`Op:Max`." + return (numpy.maximum(*inps), ) + + def _op_mul(self, x, y): + "Runtime for operator :epkg:`Op:Mul`." + return (x * y, ) + + def _op_reduceprod(self, data, axes=None, keepdims=None): + "Runtime for operator :epkg:`Op:ReduceProd`." + if axes is not None and not isinstance(axes, int): + if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: + axes = int(axes) + else: + axes = tuple(axes) if len(axes) > 0 else None + return (numpy.prod(data, axis=axes, + keepdims=keepdims, + dtype=data.dtype), ) + + def _op_reducesum(self, data, axes, keepdims=None, + noop_with_empty_axes=None): + "Runtime for operator :epkg:`Op:ReduceSum`." + if axes is None and noop_with_empty_axes: + return (data, ) + if axes is not None and not isinstance(axes, int): + if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: + axes = int(axes) + else: + axes = tuple(axes) if len(axes) > 0 else None + return (numpy.sum(data, axis=axes, + keepdims=keepdims, + dtype=data.dtype), ) + + def _op_reshape(self, x, shape): + "Runtime for operator :epkg:`Op:Reshape`." + return (x.reshape(shape), ) + + def _op_shape(self, x): + "Runtime for operator :epkg:`Op:Shape`." + return (numpy.array(list(x.shape), dtype=numpy.int64), ) + + def _op_squeeze(self, x, axes=None): + "Runtime for operator :epkg:`Op:Squeeze`." + if axes is None: + return (x, ) + if hasattr(axes, '__iter__'): + return (numpy.squeeze(x, axis=tuple(axes)), ) + return (numpy.squeeze(x, axis=axes), ) + + def _op_transpose(self, x, perm=None): + "Runtime for operator :epkg:`Op:Transpose`." + return (numpy.transpose(x, perm), ) + + def _op_unsqueeze(self, x, axes=None): + "Runtime for operator :epkg:`Op:Unsqueeze`." + if axes is None: + return (x, ) + if hasattr(axes, '__iter__'): + return (numpy.expand_dims(x, axis=tuple(axes)), ) + return (numpy.expand_dims(x, axis=axes), ) diff --git a/mlprodict/onnxrt/ops_shape/__init__.py b/mlprodict/onnxrt/ops_shape/__init__.py new file mode 100644 index 000000000..49c1fff6b --- /dev/null +++ b/mlprodict/onnxrt/ops_shape/__init__.py @@ -0,0 +1,9 @@ +""" +@file +@brief Shortcut to *ops_shape*. +""" +from ._element_wise import shape_add, shape_mul, shape_div, shape_sub + +shape_functions = { + k: v for k, v in globals() if k.startswith("shape_") +} diff --git a/mlprodict/onnxrt/ops_shape/_element_wise.py b/mlprodict/onnxrt/ops_shape/_element_wise.py new file mode 100644 index 000000000..8210bd6e6 --- /dev/null +++ b/mlprodict/onnxrt/ops_shape/_element_wise.py @@ -0,0 +1,33 @@ +""" +@file +@brief Computes shape inference for element wise operators. +""" + +def _element_wise(known_shapes, node, x, y): + """ + Infers shape for an element wise operator. + + :param known_shapes: known shapes + :param node: Onnx node + :param x: first argument + :param y: second argument + :return: + """ + raise NotImplementedError() + + +def shape_add(known_shapes, node, x, y): + "Infers shape for operator Add." + return _element_wise(known_shapes, node, x, y) + +def shape_sub(known_shapes, node, x, y): + "Infers shape for operator Sub." + return _element_wise(known_shapes, node, x, y) + +def shape_div(known_shapes, node, x, y): + "Infers shape for operator Div." + return _element_wise(known_shapes, node, x, y) + +def shape_mul(known_shapes, node, x, y): + "Infers shape for operator Mul." + return _element_wise(known_shapes, node, x, y) diff --git a/mlprodict/onnxrt/ops_shape/shape_result.py b/mlprodict/onnxrt/ops_shape/shape_result.py new file mode 100644 index 000000000..aa3c4c0fa --- /dev/null +++ b/mlprodict/onnxrt/ops_shape/shape_result.py @@ -0,0 +1,40 @@ +""" +@file +@brief Class ShapeResult +""" +from enum import Enum + +class OnnxKind(Enum): + """ + Describes a result type. + """ + Tensor = 0 + Sequence = 0 + Map = 0 + + +class ShapeResult: + """ + Contains information about shape and type of a result + in an onnx graph. + + :param shape: shape if the result is a tensor + :param dtype: element type if the result is a tensor + :param sparse: is a the tensor sparse + :param mtype: kind of the result (see class @see cl OnnxKind) + """ + def __init__(self, shape=None, dtype=None, sparse=False, + mtype=OnnxKind.Tensor): + self.mtype = mtype + self.shape = shape + self.dtype = dtype + self.sparse = sparse + + def __repr__(self): + """ + Usual + """ + return "%s(%r, %r, %r, %r)" % ( + self.__class__.__name__, self.shape, self.dtype, + self.sparse, self.mtype) + From 2f5fb4de0032016cd2ed3f1e93dd5041383462ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Sat, 5 Feb 2022 13:26:55 +0100 Subject: [PATCH 03/10] First version of ShapeInference --- _unittests/ut_npy/test_complex_scenario.py | 6 +- _unittests/ut_onnxrt/test_shape_inference.py | 33 +-- mlprodict/onnxrt/onnx_shape_inference.py | 237 ++++++------------ mlprodict/onnxrt/ops_shape/__init__.py | 23 +- mlprodict/onnxrt/ops_shape/_element_wise.py | 34 +-- mlprodict/onnxrt/ops_shape/shape_container.py | 58 +++++ mlprodict/onnxrt/ops_shape/shape_result.py | 79 +++++- 7 files changed, 271 insertions(+), 199 deletions(-) create mode 100644 mlprodict/onnxrt/ops_shape/shape_container.py diff --git a/_unittests/ut_npy/test_complex_scenario.py b/_unittests/ut_npy/test_complex_scenario.py index 94c768fc2..5df34bf79 100644 --- a/_unittests/ut_npy/test_complex_scenario.py +++ b/_unittests/ut_npy/test_complex_scenario.py @@ -195,9 +195,9 @@ def test_futr_fft_abs(self): def tf_fft(x): import tensorflow as tf # pylint: disable=E0401 - xc = tf.cast(x, tf.complex64) - xcf = tf.signal.fft(xc) - return tf.abs(xcf) + xc = tf.cast(x, tf.complex64) # pylint: disable=E1101 + xcf = tf.signal.fft(xc) # pylint: disable=E1101 + return tf.abs(xcf) # pylint: disable=E1101 try: tfx = tf_fft(x) diff --git a/_unittests/ut_onnxrt/test_shape_inference.py b/_unittests/ut_onnxrt/test_shape_inference.py index 999fef77a..8637d6dfe 100644 --- a/_unittests/ut_onnxrt/test_shape_inference.py +++ b/_unittests/ut_onnxrt/test_shape_inference.py @@ -5,30 +5,35 @@ import numpy from pyquickhelper.pycode import ExtTestCase from skl2onnx.algebra.onnx_ops import ( # pylint: disable=E0611 - OnnxAdd, OnnxTranspose, OnnxShape, OnnxPow, OnnxMatMul, OnnxGemm, - OnnxSqueeze, OnnxUnsqueeze) + OnnxAdd) from mlprodict.onnxrt import OnnxShapeInference +from mlprodict.tools import get_opset_number_from_onnx class TestOnnxShapeInference(ExtTestCase): - opsets = (10, onnx. + opsets = list(range(10, get_opset_number_from_onnx() + 1)) def test_onnx_micro_runtime(self): - opset = OnnxShapeInference.opset dtype = numpy.float32 x = numpy.array([1, 2, 4, 5, 5, 4]).astype( numpy.float32).reshape((3, 2)) - cop = OnnxAdd('X', numpy.array([1], dtype=dtype), op_version=opset) - cop4 = OnnxAdd(cop, numpy.array([2], dtype=dtype), op_version=opset, - output_names=['Y']) - model_def = cop4.to_onnx({'X': x}, target_opset=opset) - rt = OnnxShapeInference(model_def) - out = rt.run({'X': x}) - self.assertIn('X', out) - self.assertIn('Y', out) - self.assertIn('Ad_Addcst', out) - self.assertEqual(len(out), 5) + for opset in TestOnnxShapeInference.opsets: + with self.subTest(opset=opset): + cop = OnnxAdd('X', numpy.array( + [[1]], dtype=dtype), op_version=opset) + cop4 = OnnxAdd(cop, numpy.array([[2]], dtype=dtype), op_version=opset, + output_names=['Y']) + model_def = cop4.to_onnx({'X': x}, target_opset=opset) + rt = OnnxShapeInference(model_def) + out = rt.run({'X': x}) + self.assertIn('X', out) + self.assertIn('Y', out) + self.assertIn('Ad_Addcst', out) + self.assertEqual(len(out), 5) + self.assertIn( + "Ad_C0: ShapeResult([3, 2], dtype('float32'), " + "True, )", str(out)) if __name__ == "__main__": diff --git a/mlprodict/onnxrt/onnx_shape_inference.py b/mlprodict/onnxrt/onnx_shape_inference.py index c2c6ee46e..8638426bd 100644 --- a/mlprodict/onnxrt/onnx_shape_inference.py +++ b/mlprodict/onnxrt/onnx_shape_inference.py @@ -5,7 +5,11 @@ .. versionadded:: 0.9 """ import numpy -from ..onnx_tools.onnx2py_helper import _var_as_dict +from onnx.numpy_helper import to_array +from onnx.mapping import TENSOR_TYPE_TO_NP_TYPE +from .ops_shape.shape_result import ShapeResult +from .ops_shape.shape_container import ShapeContainer +from .ops_shape import shape_dispatch class OnnxShapeInference: @@ -21,173 +25,78 @@ def __init__(self, model_onnx): raise TypeError( "model_onnx is not an ONNX graph but %r." % type(model_onnx)) self.model_onnx = model_onnx + self.known_shapes_ = self._run_empty() - def run(self, inputs): + def __repr__(self): + "Usual" + return "%s(...)" % self.__class__.__name__ + + @staticmethod + def _get_shape(obj): + dtype = TENSOR_TYPE_TO_NP_TYPE[obj.type.tensor_type.elem_type] + shape = [] + for d in obj.type.tensor_type.shape.dim: + shape.append(d.dim_value if d.dim_value > 0 else d.dim_param) + return shape, dtype, False + + def _run_empty(self): """ - Computes the outputs of the graph. + Computes shape and types of all results. - :param inputs: dictionary :return: all intermediates results and output as a dictionary """ - if not isinstance(inputs, dict): - raise TypeError( - "inputs must be a dictionary not %r." % type(inputs)) - results = inputs.copy() - + known_shapes = ShapeContainer() for init in self.model_onnx.graph.initializer: - name = init.name - mat = _var_as_dict(init)['value'] - results[name] = mat - - for node in self.model_onnx.graph.node: - op_type = node.op_type - inp = [results[n] for n in node.input] - meth_name = "_op_%s" % op_type.lower() - if not hasattr(self, meth_name): + mat = to_array(init) + known_shapes.update(init.name, ShapeResult( + mat.shape, mat.dtype, sparse=False)) + + for obj in self.model_onnx.graph.input: + if obj.name in known_shapes: raise NotImplementedError( - "OnnxMicroRuntime does not implement operator %r." % op_type) - kwargs = {} - for at in node.attribute: - var = _var_as_dict(at) - kwargs[at.name] = var['value'] - out = getattr(self, meth_name)(*inp, **kwargs) - for n, o in zip(node.output, out): - results[n] = o - - return results - - ######################## - # Runtime for operators - ######################## - - def _op_add(self, x, y): - "Runtime for operator :epkg:`Op:Add`." - return (x + y, ) - - def _op_concat(self, *args, axis=None): - "Runtime for operator :epkg:`Op:Concat`." - def _preprocess(a, axis): - if axis >= len(a.shape): - new_shape = a.shape + (1, ) * (axis + 1 - len(a.shape)) - return a.reshape(new_shape) - return a - - targs = tuple(_preprocess(a, axis) for a in args) - return (numpy.concatenate(targs, axis), ) - - def _op_gemm(self, a, b, c=None, alpha=None, beta=None, - transA=False, transB=False): - "Runtime for operator :epkg:`Op:Gemm`." - - def _gemm00(a, b, c, alpha, beta): - o = numpy.dot(a, b) * alpha - if beta != 0: - o += c * beta - return o - - def _gemm01(a, b, c, alpha, beta): - o = numpy.dot(a, b.T) * alpha - if beta != 0: - o += c * beta - return o - - def _gemm10(a, b, c, alpha, beta): - o = numpy.dot(a.T, b) * alpha - if beta != 0: - o += c * beta - return o - - def _gemm11(a, b, c, alpha, beta): - o = numpy.dot(a.T, b.T) * alpha - if beta != 0: - o += c * beta - return o - - if not isinstance(transA, (int, bool, numpy.int64)): - raise TypeError( # pragma: no cover - "Unexpected type for transA: %r." % type(transA)) - if not isinstance(transB, (int, bool, numpy.int64)): - raise TypeError( # pragma: no cover - "Unexpected type for transA: %r." % type(transB)) - if transA: - fct = _gemm11 if transB else _gemm10 - else: - fct = _gemm01 if transB else _gemm00 - return (fct(a, b, c, alpha=alpha, beta=beta), ) - - def _op_gather(self, x, indices, axis=None): - "Runtime for operator :epkg:`Op:Gather`." - if not x.flags['C_CONTIGUOUS']: - x = numpy.ascontiguousarray(x) - if not indices.flags['C_CONTIGUOUS']: - indices = indices.ascontiguousarray() - return (numpy.take(x, indices, axis=axis), ) - - def _op_identity(self, x): - "Runtime for operator :epkg:`Op:Identity`." - return (x, ) - - def _op_matmul(self, x, y): - "Runtime for operator :epkg:`Op:MatMul`." - return (numpy.matmul(x, y), ) - - def _op_max(self, *inps): - "Runtime for operator :epkg:`Op:Max`." - return (numpy.maximum(*inps), ) - - def _op_mul(self, x, y): - "Runtime for operator :epkg:`Op:Mul`." - return (x * y, ) - - def _op_reduceprod(self, data, axes=None, keepdims=None): - "Runtime for operator :epkg:`Op:ReduceProd`." - if axes is not None and not isinstance(axes, int): - if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: - axes = int(axes) - else: - axes = tuple(axes) if len(axes) > 0 else None - return (numpy.prod(data, axis=axes, - keepdims=keepdims, - dtype=data.dtype), ) - - def _op_reducesum(self, data, axes, keepdims=None, - noop_with_empty_axes=None): - "Runtime for operator :epkg:`Op:ReduceSum`." - if axes is None and noop_with_empty_axes: - return (data, ) - if axes is not None and not isinstance(axes, int): - if isinstance(axes, numpy.ndarray) and len(axes.shape) == 0: - axes = int(axes) - else: - axes = tuple(axes) if len(axes) > 0 else None - return (numpy.sum(data, axis=axes, - keepdims=keepdims, - dtype=data.dtype), ) - - def _op_reshape(self, x, shape): - "Runtime for operator :epkg:`Op:Reshape`." - return (x.reshape(shape), ) - - def _op_shape(self, x): - "Runtime for operator :epkg:`Op:Shape`." - return (numpy.array(list(x.shape), dtype=numpy.int64), ) - - def _op_squeeze(self, x, axes=None): - "Runtime for operator :epkg:`Op:Squeeze`." - if axes is None: - return (x, ) - if hasattr(axes, '__iter__'): - return (numpy.squeeze(x, axis=tuple(axes)), ) - return (numpy.squeeze(x, axis=axes), ) - - def _op_transpose(self, x, perm=None): - "Runtime for operator :epkg:`Op:Transpose`." - return (numpy.transpose(x, perm), ) - - def _op_unsqueeze(self, x, axes=None): - "Runtime for operator :epkg:`Op:Unsqueeze`." - if axes is None: - return (x, ) - if hasattr(axes, '__iter__'): - return (numpy.expand_dims(x, axis=tuple(axes)), ) - return (numpy.expand_dims(x, axis=axes), ) + "Optional inputs are not implemented yet.") + shape, dtype, sparse = self._get_shape(obj) + known_shapes.update(obj.name, ShapeResult( + shape, dtype, sparse=sparse)) + + for obj in self.model_onnx.graph.output: + if obj.name in known_shapes: + raise NotImplementedError( + "Optional inputs are not implemented yet.") + shape, dtype, sparse = self._get_shape(obj) + known_shapes.update(obj.name, ShapeResult( + shape, dtype, sparse=sparse)) + + cont = True + while cont: + cont = False + for node in self.model_onnx.graph.node: + cont = cont or shape_dispatch(known_shapes, node) + + return known_shapes + + def run(self, inputs=None): + """ + Runs shape inference and type given known inputs. + + :param inputs: inputs + :return: all results + """ + if inputs is None: + return self.known_shapes_ + + known_shapes = self.known_shapes_.copy() + + cont = False + for name, obj in inputs.items(): + shape, dtype, sparse = ( + obj.shape, obj.dtype, isinstance(obj, numpy.ndarray)) + cont = cont or known_shapes.update( + name, ShapeResult(shape, dtype, sparse=sparse)) + + while cont: + cont = False + for node in self.model_onnx.graph.node: + cont = cont or shape_dispatch(known_shapes, node) + + return known_shapes diff --git a/mlprodict/onnxrt/ops_shape/__init__.py b/mlprodict/onnxrt/ops_shape/__init__.py index 49c1fff6b..d88679cb9 100644 --- a/mlprodict/onnxrt/ops_shape/__init__.py +++ b/mlprodict/onnxrt/ops_shape/__init__.py @@ -4,6 +4,25 @@ """ from ._element_wise import shape_add, shape_mul, shape_div, shape_sub -shape_functions = { - k: v for k, v in globals() if k.startswith("shape_") + +_shape_functions = { + k: v for k, v in globals().items() if k.startswith("shape_") } + + +def shape_dispatch(known_shape, node): + """ + Calls the corresponding fucntion for every node. + + :param known_shape: known_shape for all results + :param node: onnx node + :return: was *known_shape* updated or not... + """ + op_type = "shape_" + node.op_type.lower() + if op_type in _shape_functions: + return _shape_functions[op_type](known_shape, node) + raise RuntimeError( + "Unable to find a corresponding function for operator type %r " + "domain=%r among\n%s" % ( + node.op_type, node.doomain, + "\n".join(sorted(_shape_functions)))) diff --git a/mlprodict/onnxrt/ops_shape/_element_wise.py b/mlprodict/onnxrt/ops_shape/_element_wise.py index 8210bd6e6..08b7bea7b 100644 --- a/mlprodict/onnxrt/ops_shape/_element_wise.py +++ b/mlprodict/onnxrt/ops_shape/_element_wise.py @@ -2,32 +2,38 @@ @file @brief Computes shape inference for element wise operators. """ +from .shape_result import ShapeResult -def _element_wise(known_shapes, node, x, y): + +def _element_wise(known_shapes, node): """ Infers shape for an element wise operator. + The function returns but updates *known_shapes*. :param known_shapes: known shapes :param node: Onnx node - :param x: first argument - :param y: second argument - :return: + :return: updated or not """ - raise NotImplementedError() - + x = known_shapes[node.input[0]] + y = known_shapes[node.input[1]] + return known_shapes.update(node.output[0], ShapeResult.broadcast(x, y)) + -def shape_add(known_shapes, node, x, y): +def shape_add(known_shapes, node): "Infers shape for operator Add." - return _element_wise(known_shapes, node, x, y) - + return _element_wise(known_shapes, node) + + def shape_sub(known_shapes, node, x, y): "Infers shape for operator Sub." - return _element_wise(known_shapes, node, x, y) - + return _element_wise(known_shapes, node) + + def shape_div(known_shapes, node, x, y): "Infers shape for operator Div." - return _element_wise(known_shapes, node, x, y) - + return _element_wise(known_shapes, node) + + def shape_mul(known_shapes, node, x, y): "Infers shape for operator Mul." - return _element_wise(known_shapes, node, x, y) + return _element_wise(known_shapes, node) diff --git a/mlprodict/onnxrt/ops_shape/shape_container.py b/mlprodict/onnxrt/ops_shape/shape_container.py new file mode 100644 index 000000000..939599619 --- /dev/null +++ b/mlprodict/onnxrt/ops_shape/shape_container.py @@ -0,0 +1,58 @@ +""" +@file +@brief Class ShapeContainer +""" +from .shape_result import ShapeResult + + +class ShapeContainer: + """ + Stores all infered shapes. + """ + + def __init__(self): + self.shapes = dict() + + def __len__(self): + "usual" + return len(self.shapes) + + def __getitem__(self, key): + "Retrieves one shape from its name." + return self.shapes[key] + + def copy(self): + "Makes a copy." + cont = ShapeContainer() + cont.shapes = self.shapes.copy() + return cont + + def update(self, key, value): + """ + Updates one shape. Returns True if the shape was different. + """ + if not isinstance(key, str): + raise TypeError("key must be a string not %r." % type(key)) + if not isinstance(value, ShapeResult): + raise TypeError("value must be a ShapeResult not %r." % type(key)) + if key not in self.shapes: + self.shapes[key] = value + return True + if self.shapes[key] == value: + return False + self.shapes[key] = value + return True + + def __contains__(self, key): + "Operator in." + return key in self.shapes + + def __str__(self): + """ + Displays. + """ + rows = ["ShapeContainer({"] + for k, v in self.shapes.items(): + rows.append(" %s: %r" % (k, v)) + rows.append("})") + return "\n".join(rows) diff --git a/mlprodict/onnxrt/ops_shape/shape_result.py b/mlprodict/onnxrt/ops_shape/shape_result.py index aa3c4c0fa..eefee03bb 100644 --- a/mlprodict/onnxrt/ops_shape/shape_result.py +++ b/mlprodict/onnxrt/ops_shape/shape_result.py @@ -4,10 +4,18 @@ """ from enum import Enum + +class ShapeInferenceException(RuntimeError): + """ + Raised when shape inference fails. + """ + pass + + class OnnxKind(Enum): """ Describes a result type. - """ + """ Tensor = 0 Sequence = 0 Map = 0 @@ -23,13 +31,14 @@ class ShapeResult: :param sparse: is a the tensor sparse :param mtype: kind of the result (see class @see cl OnnxKind) """ + def __init__(self, shape=None, dtype=None, sparse=False, mtype=OnnxKind.Tensor): self.mtype = mtype self.shape = shape self.dtype = dtype self.sparse = sparse - + def __repr__(self): """ Usual @@ -38,3 +47,69 @@ def __repr__(self): self.__class__.__name__, self.shape, self.dtype, self.sparse, self.mtype) + def __eq__(self, shape): + """ + Tells if two shapes are identical. + """ + return (self.mtype == shape.mtype and self.shape == shape.shape and + self.dtype == shape.dtype and self.sparse == shape.sparse) + + def n_dims(self): + """ + Returns the number of dimensions if it is a tensor. + Raises an exception otherwise. + """ + if self.mtype != OnnxKind.Tensor: + raise ShapeInferenceException( + "This shape is not a tensor %r." % self) + return len(self.shape) + + @staticmethod + def broadcast(sh1, sh2): + """ + Broadcasts dimensions for an element wise operator. + + :param sh1: ShapeResult + :param sh2: ShapeResult + :return: ShapeResult + """ + if not isinstance(sh1, ShapeResult): + raise TypeError("Unexpected type for sh1 %r." % type(sh1)) + if not isinstance(sh2, ShapeResult): + raise TypeError("Unexpected type for sh2 %r." % type(sh2)) + if sh1.mtype != OnnxKind.Tensor: + raise TypeError("sh1 must be a tensor not %r." % sh1.mtype) + if sh2.mtype != OnnxKind.Tensor: + raise TypeError("sh2 must be a tensor not %r." % sh2.mtype) + if sh1.n_dims() != sh2.n_dims(): + raise ShapeInferenceException( + "Broadcasting is only implemented for shape of the same " + "size, shapes are %r and %r." % (sh1, sh2)) + if sh1.dtype != sh2.dtype: + raise ShapeInferenceException( + "Cannot broadcast shapes %r and %r (dtypes)." + "" % (sh1, sh2)) + shape = [] + for a, b in zip(sh1.shape, sh2.shape): + if isinstance(a, int) and isinstance(b, int): + if a != b: + if min(a, b) == 1: + d = max(a, b) + else: + raise ShapeInferenceException( + "Cannot broadcast shapes %r and %r (dimensions)." + "" % (sh1, sh2)) + else: + d = a + elif isinstance(a, int): + d = a + elif isinstance(b, int): + d = b + elif a == b: + d = a + else: + raise ShapeInferenceException( + "Cannot broadcast shapes %r and %r." % (sh1, sh2)) + shape.append(d) + return ShapeResult(shape, sh1.dtype, sh1.sparse or sh2.sparse, + sh1.mtype) From 4e5b6c2efb0566611596fbc11e2927b4f4ba29db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Mon, 7 Feb 2022 00:21:58 +0100 Subject: [PATCH 04/10] improve behaviour for shape inference --- _unittests/ut_module/test_code_style.py | 3 +- _unittests/ut_onnxrt/test_shape_inference.py | 34 ++++++++++++ mlprodict/onnxrt/onnx_shape_inference.py | 19 +++++-- mlprodict/onnxrt/ops_shape/shape_container.py | 48 +++++++++++++++++ mlprodict/onnxrt/ops_shape/shape_result.py | 54 ++++++++++++++++--- 5 files changed, 146 insertions(+), 12 deletions(-) diff --git a/_unittests/ut_module/test_code_style.py b/_unittests/ut_module/test_code_style.py index c78838c98..e7b068194 100644 --- a/_unittests/ut_module/test_code_style.py +++ b/_unittests/ut_module/test_code_style.py @@ -35,7 +35,8 @@ def test_style_test(self): check_pep8(test, fLOG=fLOG, neg_pattern="temp_.*", pylint_ignore=('C0103', 'C1801', 'R0201', 'R1705', 'W0108', 'W0613', 'C0111', 'W0107', 'C0415', 'R1728', 'C0209', - 'R1721', 'C0302', 'C0411', 'R1735', 'W1514'), + 'R1721', 'C0302', 'C0411', 'R1735', 'W1514', + 'C0200', 'E1101', 'W0212'), skip=["Instance of 'tuple' has no ", "R1720", 'if __name__ == "__main__":', diff --git a/_unittests/ut_onnxrt/test_shape_inference.py b/_unittests/ut_onnxrt/test_shape_inference.py index 8637d6dfe..f538ab26f 100644 --- a/_unittests/ut_onnxrt/test_shape_inference.py +++ b/_unittests/ut_onnxrt/test_shape_inference.py @@ -3,10 +3,13 @@ """ import unittest import numpy +from onnx.shape_inference import infer_shapes from pyquickhelper.pycode import ExtTestCase from skl2onnx.algebra.onnx_ops import ( # pylint: disable=E0611 OnnxAdd) from mlprodict.onnxrt import OnnxShapeInference +from mlprodict.onnxrt.ops_shape.shape_result import ShapeResult +from mlprodict.plotting.text_plot import onnx_simple_text_plot from mlprodict.tools import get_opset_number_from_onnx @@ -14,6 +17,26 @@ class TestOnnxShapeInference(ExtTestCase): opsets = list(range(10, get_opset_number_from_onnx() + 1)) + def check_infer_shapes(self, onx, out, rt): + onnx_shapes = infer_shapes(onx) + inferred = onnx_shapes.graph.value_info # pylint: disable= + for data in inferred: + if data.name not in out: + raise AssertionError("Name %r not found." % data.name) + shape, dtype, sparse = OnnxShapeInference._get_shape(data) # pylint: disable=W0212 + for i in range(len(shape)): + if not isinstance(shape[i], str): + continue + if shape[i].startswith('unk_'): + shape[i] = shape[i][4:] + res = ShapeResult(shape, dtype, sparse) + if res != out[data.name]: + raise AssertionError( + "Unexpected differences for name %r:\nexp: %r\ngot: %r" + "\n-----\n%s" % ( + data.name, res, out[data.name], + onnx_simple_text_plot(onx))) + def test_onnx_micro_runtime(self): dtype = numpy.float32 x = numpy.array([1, 2, 4, 5, 5, 4]).astype( @@ -34,6 +57,17 @@ def test_onnx_micro_runtime(self): self.assertIn( "Ad_C0: ShapeResult([3, 2], dtype('float32'), " "True, )", str(out)) + self.check_infer_shapes(model_def, rt.run(), rt) + cons = rt.known_shapes_.get_all_constraints() + self.assertEqual(len(cons), 1) + self.assertEqual(list(cons), ['_0']) + self.assertEqual(len(cons['_0']), 1) + cst = cons['_0'][0] + self.assertEqual(cst.name, '_0') + self.assertEqual(cst.values, {1, '_0'}) + self.assertEqual( + rt.known_shapes_.names, + {'_0': ('', 'X', 0), '_1': ('', 'Y', 0)}) if __name__ == "__main__": diff --git a/mlprodict/onnxrt/onnx_shape_inference.py b/mlprodict/onnxrt/onnx_shape_inference.py index 8638426bd..1b746a0b8 100644 --- a/mlprodict/onnxrt/onnx_shape_inference.py +++ b/mlprodict/onnxrt/onnx_shape_inference.py @@ -32,11 +32,18 @@ def __repr__(self): return "%s(...)" % self.__class__.__name__ @staticmethod - def _get_shape(obj): + def _get_shape(obj, known_shapes=None, result_name=None): dtype = TENSOR_TYPE_TO_NP_TYPE[obj.type.tensor_type.elem_type] shape = [] - for d in obj.type.tensor_type.shape.dim: - shape.append(d.dim_value if d.dim_value > 0 else d.dim_param) + for dimi, d in enumerate(obj.type.tensor_type.shape.dim): + v = d.dim_value if d.dim_value > 0 else d.dim_param + if v in ('', None): + if known_shapes is None or result_name is None: + raise RuntimeError( # pragma: no cover + "known_shapes must be specified if " + "a dimension is not.") + v = known_shapes.get_new_name(v, result_name, dimi) + shape.append(v) return shape, dtype, False def _run_empty(self): @@ -55,7 +62,8 @@ def _run_empty(self): if obj.name in known_shapes: raise NotImplementedError( "Optional inputs are not implemented yet.") - shape, dtype, sparse = self._get_shape(obj) + shape, dtype, sparse = self._get_shape( + obj, known_shapes, result_name=obj.name) known_shapes.update(obj.name, ShapeResult( shape, dtype, sparse=sparse)) @@ -63,7 +71,8 @@ def _run_empty(self): if obj.name in known_shapes: raise NotImplementedError( "Optional inputs are not implemented yet.") - shape, dtype, sparse = self._get_shape(obj) + shape, dtype, sparse = self._get_shape( + obj, known_shapes, result_name=obj.name) known_shapes.update(obj.name, ShapeResult( shape, dtype, sparse=sparse)) diff --git a/mlprodict/onnxrt/ops_shape/shape_container.py b/mlprodict/onnxrt/ops_shape/shape_container.py index 939599619..1e7bc394a 100644 --- a/mlprodict/onnxrt/ops_shape/shape_container.py +++ b/mlprodict/onnxrt/ops_shape/shape_container.py @@ -12,6 +12,8 @@ class ShapeContainer: def __init__(self): self.shapes = dict() + self.names = dict() + self.names_rev = dict() def __len__(self): "usual" @@ -25,6 +27,8 @@ def copy(self): "Makes a copy." cont = ShapeContainer() cont.shapes = self.shapes.copy() + cont.names = self.names.copy() + cont.names_rev = {k: v.copy() for k, v in self.names_rev.items()} return cont def update(self, key, value): @@ -56,3 +60,47 @@ def __str__(self): rows.append(" %s: %r" % (k, v)) rows.append("})") return "\n".join(rows) + + def get_new_name(self, name, result_name, dim): + """ + Returns a variable name when a dimension is not + specified. + """ + if name is not None and not isinstance(name, str): + raise TypeError("name must be string not %r." % name) + if name is None: + name = '' + if name == '' or name not in self.names: + i = 0 + new_name = "%s_%d" % (name, i) + while new_name in self.names: + i += 1 + new_name = "%s_%d" % (name, i) + self.names[new_name] = (name, result_name, dim) + if name not in self.names_rev: + self.names_rev[name] = [] + self.names_rev[name].append(new_name) + return new_name + val = self.names_rev[name] + if len(val) != 1: + raise RuntimeError( + "Name %r has more than one correspondance (%r)." % ( + name, val)) + return val[0] + + def get_all_constraints(self): + """ + Gathers all constraintes while inferring the shape. + """ + cons = {} + for _, v in self.shapes.items(): + if v.constraints is not None: + for c in v.constraints: + if c.name not in cons: + cons[c.name] = [] + cons[c.name].append(c) + for _, v in cons.items(): + if len(v) > 1: + v[0].merge(v[1:]) + del v[1:] + return cons diff --git a/mlprodict/onnxrt/ops_shape/shape_result.py b/mlprodict/onnxrt/ops_shape/shape_result.py index eefee03bb..f8c52a8e0 100644 --- a/mlprodict/onnxrt/ops_shape/shape_result.py +++ b/mlprodict/onnxrt/ops_shape/shape_result.py @@ -21,6 +21,36 @@ class OnnxKind(Enum): Map = 0 +class ShapeConstraint: + """ + One constraint. + + :param name: variable name + :param values: set of possible values + """ + + def __init__(self, name, values): + if name == '?': + raise ValueError("Name cannot be '?'.") + self.name = name + self.values = values + + def __repr__(self): + "usual" + return "%s(%r, %r)" % ( + self.__class__.__name__, self.name, self.values) + + def merge(self, cst): + """ + Merges this constraint with *cst* into this one. + """ + if isinstance(cst, list): + for c in cst: + self.merge(c) + return + self.values = self.values.intersection(cst.values) + + class ShapeResult: """ Contains information about shape and type of a result @@ -30,14 +60,21 @@ class ShapeResult: :param dtype: element type if the result is a tensor :param sparse: is a the tensor sparse :param mtype: kind of the result (see class @see cl OnnxKind) + :param constraints: list of constraints applying on variables """ def __init__(self, shape=None, dtype=None, sparse=False, - mtype=OnnxKind.Tensor): + mtype=OnnxKind.Tensor, constraints=None): self.mtype = mtype - self.shape = shape + self.shape = list(shape) self.dtype = dtype self.sparse = sparse + for i in range(0, len(self.shape)): # pylint: disable=C0200 + if shape[i] in ('', None, '?'): + raise ValueError( + "All dimensions must an int or a variable name, " + "%s is not." % (shape, )) + self.constraints = constraints def __repr__(self): """ @@ -89,6 +126,8 @@ def broadcast(sh1, sh2): raise ShapeInferenceException( "Cannot broadcast shapes %r and %r (dtypes)." "" % (sh1, sh2)) + + constraints = [] shape = [] for a, b in zip(sh1.shape, sh2.shape): if isinstance(a, int) and isinstance(b, int): @@ -102,14 +141,17 @@ def broadcast(sh1, sh2): else: d = a elif isinstance(a, int): - d = a - elif isinstance(b, int): d = b + constraints.append(ShapeConstraint(b, {1, a, b})) + elif isinstance(b, int): + d = a + constraints.append(ShapeConstraint(a, {1, b, a})) elif a == b: d = a else: raise ShapeInferenceException( "Cannot broadcast shapes %r and %r." % (sh1, sh2)) shape.append(d) - return ShapeResult(shape, sh1.dtype, sh1.sparse or sh2.sparse, - sh1.mtype) + res = ShapeResult(shape, sh1.dtype, sh1.sparse or sh2.sparse, + sh1.mtype, constraints) + return res From ce44895679d896c1d94da6dbfc62cb6af97401e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Mon, 7 Feb 2022 17:57:47 +0100 Subject: [PATCH 05/10] revisit the constraint mechanism --- _unittests/ut_onnxrt/test_shape_inference.py | 72 ++++++++++-- mlprodict/onnxrt/ops_shape/shape_container.py | 21 +++- mlprodict/onnxrt/ops_shape/shape_result.py | 111 ++++++++++++++++-- mlprodict/sklapi/onnx_tokenizer.py | 17 ++- 4 files changed, 197 insertions(+), 24 deletions(-) diff --git a/_unittests/ut_onnxrt/test_shape_inference.py b/_unittests/ut_onnxrt/test_shape_inference.py index f538ab26f..9267c3317 100644 --- a/_unittests/ut_onnxrt/test_shape_inference.py +++ b/_unittests/ut_onnxrt/test_shape_inference.py @@ -7,8 +7,10 @@ from pyquickhelper.pycode import ExtTestCase from skl2onnx.algebra.onnx_ops import ( # pylint: disable=E0611 OnnxAdd) +from skl2onnx.common.data_types import FloatTensorType from mlprodict.onnxrt import OnnxShapeInference -from mlprodict.onnxrt.ops_shape.shape_result import ShapeResult +from mlprodict.onnxrt.ops_shape.shape_result import ( + ShapeResult, ShapeConstraint, ShapeConstraintList) from mlprodict.plotting.text_plot import onnx_simple_text_plot from mlprodict.tools import get_opset_number_from_onnx @@ -23,7 +25,8 @@ def check_infer_shapes(self, onx, out, rt): for data in inferred: if data.name not in out: raise AssertionError("Name %r not found." % data.name) - shape, dtype, sparse = OnnxShapeInference._get_shape(data) # pylint: disable=W0212 + shape, dtype, sparse = OnnxShapeInference._get_shape( + data) # pylint: disable=W0212 for i in range(len(shape)): if not isinstance(shape[i], str): continue @@ -37,7 +40,16 @@ def check_infer_shapes(self, onx, out, rt): data.name, res, out[data.name], onnx_simple_text_plot(onx))) - def test_onnx_micro_runtime(self): + def test_shape_constraint(self): + sh1 = ShapeConstraint('_1', {1, 2}) + sh2 = ShapeConstraint('_1', {1, 2}) + self.assertEqual(sh1, sh2) + shl = ShapeConstraintList() + shl.append(sh1) + self.assertIn(sh1, shl) + self.assertIn(sh2, shl) + + def test_onnx_shape_inference(self): dtype = numpy.float32 x = numpy.array([1, 2, 4, 5, 5, 4]).astype( numpy.float32).reshape((3, 2)) @@ -55,20 +67,62 @@ def test_onnx_micro_runtime(self): self.assertIn('Ad_Addcst', out) self.assertEqual(len(out), 5) self.assertIn( - "Ad_C0: ShapeResult([3, 2], dtype('float32'), " - "True, )", str(out)) + "'Ad_C0': ShapeResult(['_0', 2], dtype('float32'), " + "False, ", str(out)) self.check_infer_shapes(model_def, rt.run(), rt) cons = rt.known_shapes_.get_all_constraints() - self.assertEqual(len(cons), 1) - self.assertEqual(list(cons), ['_0']) + self.assertEqual(len(cons), 2) + self.assertEqual(list(cons), ['_0', '_1']) self.assertEqual(len(cons['_0']), 1) cst = cons['_0'][0] self.assertEqual(cst.name, '_0') - self.assertEqual(cst.values, {1, '_0'}) + self.assertEqual(cst.values, {3}) self.assertEqual( rt.known_shapes_.names, {'_0': ('', 'X', 0), '_1': ('', 'Y', 0)}) + def test_onnx_shape_inference_missing(self): + dtype = numpy.float32 + x = numpy.array([1, 2, 4, 5, 5, 4]).astype( + numpy.float32).reshape((3, 2)) + for opset in TestOnnxShapeInference.opsets[-1:]: + with self.subTest(opset=opset): + cop = OnnxAdd('X', numpy.array( + [[1]], dtype=dtype), op_version=opset) + cop4 = OnnxAdd(cop, numpy.array([[2, 4]], dtype=dtype), op_version=opset, + output_names=['Y']) + model_def = cop4.to_onnx( + {'X': FloatTensorType([None, None])}, + {'Y': FloatTensorType([None, None])}, + target_opset=opset) + rt = OnnxShapeInference(model_def) + out = rt.run({'X': x}) + self.assertIn('X', out) + self.assertIn('Y', out) + self.assertIn('Ad_Addcst', out) + self.assertEqual(len(out), 5) + self.assertIn( + "'Ad_C0': ShapeResult(['_0', '_1'], dtype('float32'), " + "False, ", str(out)) + out = rt.run() + self.assertIn( + "'Y': ShapeResult(['_2', '_3']", str(out)) + self.check_infer_shapes(model_def, rt.run(), rt) + cons = rt.known_shapes_.get_all_constraints() + self.assertEqual(len(rt.known_shapes_.names), 4) + self.assertEqual(set(rt.known_shapes_.names), + {'_0', '_1', '_2', '_3'}) + self.assertEqual(len(cons), 4) + self.assertEqual(list(cons), ['_0', '_1', '_2', '_3']) + self.assertEqual(len(cons['_1']), 1) + cst = cons['_1'][0] + self.assertEqual(cst.name, '_1') + self.assertEqual(cst.values, {2}) + self.assertEqual( + rt.known_shapes_.names, + {'_0': ('', 'X', 0), '_1': ('', 'X', 1), + '_2': ('', 'Y', 0), '_3': ('', 'Y', 1)}) + if __name__ == "__main__": - unittest.main() + unittest.main(verbosity=2) diff --git a/mlprodict/onnxrt/ops_shape/shape_container.py b/mlprodict/onnxrt/ops_shape/shape_container.py index 1e7bc394a..c6cf1f16f 100644 --- a/mlprodict/onnxrt/ops_shape/shape_container.py +++ b/mlprodict/onnxrt/ops_shape/shape_container.py @@ -42,10 +42,8 @@ def update(self, key, value): if key not in self.shapes: self.shapes[key] = value return True - if self.shapes[key] == value: - return False - self.shapes[key] = value - return True + r = self.shapes[key].merge(value) + return r def __contains__(self, key): "Operator in." @@ -57,8 +55,19 @@ def __str__(self): """ rows = ["ShapeContainer({"] for k, v in self.shapes.items(): - rows.append(" %s: %r" % (k, v)) - rows.append("})") + rows.append(" %r: %r" % (k, v)) + rows.append("}, names={") + for k, v in self.names.items(): + rows.append(" %r: %r" % (k, v)) + cst = self.get_all_constraints() + if len(cst) > 0: + rows.append("}, constraint={") + for c, v in cst.items(): + rows.append(" %r: %r" % (c, v)) + rows.append("})") + else: + rows.append("})") + return "\n".join(rows) def get_new_name(self, name, result_name, dim): diff --git a/mlprodict/onnxrt/ops_shape/shape_result.py b/mlprodict/onnxrt/ops_shape/shape_result.py index f8c52a8e0..e6d7463fa 100644 --- a/mlprodict/onnxrt/ops_shape/shape_result.py +++ b/mlprodict/onnxrt/ops_shape/shape_result.py @@ -35,6 +35,14 @@ def __init__(self, name, values): self.name = name self.values = values + def __eq__(self, other): + "usual" + if self.name != other.name: + return False + if self.values != other.values: + return False + return True + def __repr__(self): "usual" return "%s(%r, %r)" % ( @@ -51,6 +59,35 @@ def merge(self, cst): self.values = self.values.intersection(cst.values) +class ShapeConstraintList: + """ + A list of ShapeConstraint. + """ + + def __init__(self): + self.csts = [] + + def __contains__(self, cst): + for a in self.csts: + if cst == a: + return True + return False + + def append(self, cst): + "Appends a new constraint to the list." + self.csts.append(cst) + + def __repr__(self): + return "ShapeConstraintList(%r)" % self.csts + + def __iter__(self): + for c in self.csts: + yield c + + def __len__(self): + return len(self.csts) + + class ShapeResult: """ Contains information about shape and type of a result @@ -74,15 +111,21 @@ def __init__(self, shape=None, dtype=None, sparse=False, raise ValueError( "All dimensions must an int or a variable name, " "%s is not." % (shape, )) - self.constraints = constraints + if constraints is None: + self.constraints = ShapeConstraintList() + elif isinstance(constraints, ShapeConstraintList): + self.constraints = constraints + else: + raise TypeError( + "constraints must be of type(ShapeConstraintList).") def __repr__(self): """ Usual """ - return "%s(%r, %r, %r, %r)" % ( + return "%s(%r, %r, %r, %r, constraints=%r)" % ( self.__class__.__name__, self.shape, self.dtype, - self.sparse, self.mtype) + self.sparse, self.mtype, self.constraints) def __eq__(self, shape): """ @@ -101,6 +144,52 @@ def n_dims(self): "This shape is not a tensor %r." % self) return len(self.shape) + def merge(self, other_result): + """ + Merges constraints from *other_results* into *self*. + """ + if (self.mtype != other_result.mtype or + self.dtype != other_result.dtype or + self.sparse != other_result.sparse): + raise RuntimeError( + "Unable to merge %r and %r." % (self, other_result)) + if len(self.shape) != len(other_result.shape): + raise RuntimeError( + "Length mismatch, unable to merge %r and %r." % ( + self, other_result)) + updated = False + if other_result.constraints is not None: + for c in other_result.constraints: + if c not in self.constraints: + self.constraints.append(c) + updated = True + for a, b in zip(self.shape, other_result.shape): + if a == b: + continue + if isinstance(a, int) and isinstance(b, int): + raise RuntimeError( + "Inconsistancy between %r and %r." % ( + self, other_result)) + elif isinstance(a, str): + c = ShapeConstraint(a, {b}) + if c not in self.constraints: + updated = True + self.constraints.append(c) + elif isinstance(b, str): + c = ShapeConstraint(b, {a}) + if c not in self.constraints: + updated = True + self.constraints.append(c) + else: + raise NotImplementedError( + "Merge not implemented between %r and %r." % ( + self, other_result)) + if len(self.constraints) > 4: + raise RuntimeError( + "The number of constraints should not that many (%r)." % ( + self.constraints)) + return updated + @staticmethod def broadcast(sh1, sh2): """ @@ -127,7 +216,7 @@ def broadcast(sh1, sh2): "Cannot broadcast shapes %r and %r (dtypes)." "" % (sh1, sh2)) - constraints = [] + constraints = ShapeConstraintList() shape = [] for a, b in zip(sh1.shape, sh2.shape): if isinstance(a, int) and isinstance(b, int): @@ -141,11 +230,17 @@ def broadcast(sh1, sh2): else: d = a elif isinstance(a, int): - d = b - constraints.append(ShapeConstraint(b, {1, a, b})) + if a != 1: + d = a + constraints.append(ShapeConstraint(b, {1, a})) + else: + d = b elif isinstance(b, int): - d = a - constraints.append(ShapeConstraint(a, {1, b, a})) + if b != 1: + d = b + constraints.append(ShapeConstraint(a, {1, b})) + else: + d = a elif a == b: d = a else: diff --git a/mlprodict/sklapi/onnx_tokenizer.py b/mlprodict/sklapi/onnx_tokenizer.py index c993273ca..c45699167 100644 --- a/mlprodict/sklapi/onnx_tokenizer.py +++ b/mlprodict/sklapi/onnx_tokenizer.py @@ -11,7 +11,10 @@ from onnx import helper, TensorProto, load from onnx.defs import onnx_opset_version from onnxruntime import InferenceSession, SessionOptions -from onnxruntime_extensions import get_library_path +try: + from onnxruntime_extensions import get_library_path +except ImportError: + get_library_path = None class SentencePieceTokenizerTransformer(BaseEstimator, TransformerMixin): @@ -60,6 +63,9 @@ def __init__(self, model, nbest_size=1, alpha=0.5, reverse=False, self.add_bos = add_bos self.add_eos = add_eos self.opset = opset + if get_library_path is None: + raise ImportError( + "onnxruntime_extensions is not installed.") def __getstate__(self): state = BaseEstimator.__getstate__(self) @@ -68,6 +74,9 @@ def __getstate__(self): return state def __setstate__(self, state): + if get_library_path is None: + raise ImportError( + "onnxruntime_extensions is not installed.") state['onnx_'] = load(BytesIO(state['onnx_'])) BaseEstimator.__setstate__(self, state) so = SessionOptions() @@ -171,6 +180,9 @@ def __init__(self, vocab, merges, padding_length=-1, opset=None): self.merges = merges self.padding_length = padding_length self.opset = opset + if get_library_path is None: + raise ImportError( + "onnxruntime_extensions is not installed.") def __getstate__(self): state = BaseEstimator.__getstate__(self) @@ -179,6 +191,9 @@ def __getstate__(self): return state def __setstate__(self, state): + if get_library_path is None: + raise ImportError( + "onnxruntime_extensions is not installed.") state['onnx_'] = load(BytesIO(state['onnx_'])) BaseEstimator.__setstate__(self, state) so = SessionOptions() From 625a023ba543a87bf7cdfa7e00f61ca9a2059bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Mon, 7 Feb 2022 23:25:06 +0100 Subject: [PATCH 06/10] fix unit test --- _unittests/ut_onnxrt/test_shape_inference.py | 8 ++++---- mlprodict/onnxrt/onnx_shape_inference.py | 2 +- mlprodict/onnxrt/ops_shape/shape_result.py | 17 ++++++++++++++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/_unittests/ut_onnxrt/test_shape_inference.py b/_unittests/ut_onnxrt/test_shape_inference.py index 9267c3317..ed427ec6e 100644 --- a/_unittests/ut_onnxrt/test_shape_inference.py +++ b/_unittests/ut_onnxrt/test_shape_inference.py @@ -67,8 +67,8 @@ def test_onnx_shape_inference(self): self.assertIn('Ad_Addcst', out) self.assertEqual(len(out), 5) self.assertIn( - "'Ad_C0': ShapeResult(['_0', 2], dtype('float32'), " - "False, ", str(out)) + "'Ad_C0': ShapeResult(['_0', 2], dtype('float32')", + str(out)) self.check_infer_shapes(model_def, rt.run(), rt) cons = rt.known_shapes_.get_all_constraints() self.assertEqual(len(cons), 2) @@ -102,8 +102,8 @@ def test_onnx_shape_inference_missing(self): self.assertIn('Ad_Addcst', out) self.assertEqual(len(out), 5) self.assertIn( - "'Ad_C0': ShapeResult(['_0', '_1'], dtype('float32'), " - "False, ", str(out)) + "'Ad_C0': ShapeResult(['_0', '_1'], dtype('float32'))", + str(out)) out = rt.run() self.assertIn( "'Y': ShapeResult(['_2', '_3']", str(out)) diff --git a/mlprodict/onnxrt/onnx_shape_inference.py b/mlprodict/onnxrt/onnx_shape_inference.py index 1b746a0b8..86111fd08 100644 --- a/mlprodict/onnxrt/onnx_shape_inference.py +++ b/mlprodict/onnxrt/onnx_shape_inference.py @@ -99,7 +99,7 @@ def run(self, inputs=None): cont = False for name, obj in inputs.items(): shape, dtype, sparse = ( - obj.shape, obj.dtype, isinstance(obj, numpy.ndarray)) + obj.shape, obj.dtype, not isinstance(obj, numpy.ndarray)) cont = cont or known_shapes.update( name, ShapeResult(shape, dtype, sparse=sparse)) diff --git a/mlprodict/onnxrt/ops_shape/shape_result.py b/mlprodict/onnxrt/ops_shape/shape_result.py index e6d7463fa..47e600699 100644 --- a/mlprodict/onnxrt/ops_shape/shape_result.py +++ b/mlprodict/onnxrt/ops_shape/shape_result.py @@ -123,9 +123,20 @@ def __repr__(self): """ Usual """ - return "%s(%r, %r, %r, %r, constraints=%r)" % ( - self.__class__.__name__, self.shape, self.dtype, - self.sparse, self.mtype, self.constraints) + if len(self.constraints) > 0: + return "%s(%r, %r, %r, sparse=%r, constraints=%r)" % ( + self.__class__.__name__, self.shape, self.dtype, + self.sparse, self.mtype, self.constraints) + if self.mtype != OnnxKind.Tensor: + return "%s(%r, %r, sparse=%r, mtype=%r)" % ( + self.__class__.__name__, self.shape, self.dtype, + self.sparse, self.mtype) + if self.sparse: + return "%s(%r, %r,sparse=%r)" % ( + self.__class__.__name__, self.shape, self.dtype, + self.sparse) + return "%s(%r, %r)" % ( + self.__class__.__name__, self.shape, self.dtype) def __eq__(self, shape): """ From 15ae53fb409140fade8e5a773c84aa7affbf4775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Tue, 8 Feb 2022 16:03:37 +0100 Subject: [PATCH 07/10] new update --- _doc/sphinxdoc/source/api/onnxrt.rst | 38 +++---- _unittests/ut_onnxrt/test_shape_inference.py | 8 +- mlprodict/onnxrt/onnx_shape_inference.py | 7 +- mlprodict/onnxrt/ops_shape/shape_container.py | 103 +++++++++++++++++- mlprodict/onnxrt/ops_shape/shape_result.py | 32 +++++- 5 files changed, 159 insertions(+), 29 deletions(-) diff --git a/_doc/sphinxdoc/source/api/onnxrt.rst b/_doc/sphinxdoc/source/api/onnxrt.rst index d76aa1a66..8a780926f 100644 --- a/_doc/sphinxdoc/source/api/onnxrt.rst +++ b/_doc/sphinxdoc/source/api/onnxrt.rst @@ -23,6 +23,22 @@ on the following operators :ref:`l-onnx-runtime-operators`. .. autosignature:: mlprodict.onnxrt.onnx_micro_inference.OnnxMicroRuntime :members: run +The following is technically implemented as a runtime but it does +shape inference. + +.. autosignature:: mlprodict.onnxrt.onnx_shape_inference.OnnxShapeInference + :members: run + +The execution produces a result of type: + +.. autosignature:: mlprodict.onnxrt.ops_shape.shape_container.ShapeContainer + :members: get + +Methods `get` returns a dictionary mapping result name and the following type: + +.. autosignature:: mlprodict.onnxrt.ops_shape.shape_result.ShapeResult + :members: + Python to ONNX ++++++++++++++ @@ -122,25 +138,3 @@ C++ classes .. autosignature:: mlprodict.onnxrt.ops_cpu._op_onnx_numpy.topk_element_fetch_float .. autosignature:: mlprodict.onnxrt.ops_cpu._op_onnx_numpy.topk_element_fetch_int64 - -Shapes -++++++ - -The computation of the predictions through epkg:`ONNX` may -be optimized if the shape of every nodes is known. For example, -one possible optimisation is to do inplace computation every time -it is possible but this is only possible if the size of -the input and output are the same. We could compute the predictions -for a sample and check the sizes are the same -but that could be luck. We could also guess from a couple of samples -with different sizes and assume sizes and polynomial functions -of the input size. But in rare occasions, that could be luck too. -So one way of doing it is to implement a method -:meth:`_set_shape_inference_runtime -` -which works the same say as method :meth:`_run_sequence_runtime -` -but handles shapes instead. Following class tries to implement -a way to keep track of shape along the shape. - -.. autosignature:: mlprodict.onnxrt.shape_object.ShapeObject diff --git a/_unittests/ut_onnxrt/test_shape_inference.py b/_unittests/ut_onnxrt/test_shape_inference.py index ed427ec6e..4b64ae94a 100644 --- a/_unittests/ut_onnxrt/test_shape_inference.py +++ b/_unittests/ut_onnxrt/test_shape_inference.py @@ -49,7 +49,7 @@ def test_shape_constraint(self): self.assertIn(sh1, shl) self.assertIn(sh2, shl) - def test_onnx_shape_inference(self): + def _test_onnx_shape_inference(self): dtype = numpy.float32 x = numpy.array([1, 2, 4, 5, 5, 4]).astype( numpy.float32).reshape((3, 2)) @@ -122,6 +122,12 @@ def test_onnx_shape_inference_missing(self): rt.known_shapes_.names, {'_0': ('', 'X', 0), '_1': ('', 'X', 1), '_2': ('', 'Y', 0), '_3': ('', 'Y', 1)}) + get = out.get() + self.assertEqual(get['X'].shape, get['Y'].shape) + self.assertEqual(get['Ad_C0'].shape, get['Y'].shape) + self.assertEqual(get['Ad_C0'].shape[1], 2) + self.assertEqual(len(get['Ad_C0'].shape), 2) + self.assertIsInstance(get['Ad_C0'].shape[0], str) if __name__ == "__main__": diff --git a/mlprodict/onnxrt/onnx_shape_inference.py b/mlprodict/onnxrt/onnx_shape_inference.py index 86111fd08..570d418ba 100644 --- a/mlprodict/onnxrt/onnx_shape_inference.py +++ b/mlprodict/onnxrt/onnx_shape_inference.py @@ -91,10 +91,10 @@ def run(self, inputs=None): :param inputs: inputs :return: all results """ - if inputs is None: - return self.known_shapes_ - known_shapes = self.known_shapes_.copy() + if inputs is None: + known_shapes.resolve() + return known_shapes cont = False for name, obj in inputs.items(): @@ -108,4 +108,5 @@ def run(self, inputs=None): for node in self.model_onnx.graph.node: cont = cont or shape_dispatch(known_shapes, node) + known_shapes.resolve() return known_shapes diff --git a/mlprodict/onnxrt/ops_shape/shape_container.py b/mlprodict/onnxrt/ops_shape/shape_container.py index c6cf1f16f..6dff218d0 100644 --- a/mlprodict/onnxrt/ops_shape/shape_container.py +++ b/mlprodict/onnxrt/ops_shape/shape_container.py @@ -7,7 +7,14 @@ class ShapeContainer: """ - Stores all infered shapes. + Stores all infered shapes as @see cl ShapeResult. + + Attributes: + + * `shapes`: dictionary `{ result name: ShapeResult }` + * `names`: some dimensions are unknown and represented as + variables, this dictionary keeps track of them + * `names_rev`: reverse dictionary of `names` """ def __init__(self): @@ -99,7 +106,7 @@ def get_new_name(self, name, result_name, dim): def get_all_constraints(self): """ - Gathers all constraintes while inferring the shape. + Gathers all constraints. """ cons = {} for _, v in self.shapes.items(): @@ -113,3 +120,95 @@ def get_all_constraints(self): v[0].merge(v[1:]) del v[1:] return cons + + def get(self): + """ + Returns the value of attribute `resolved_` + (method `resolve()` must have been called first). + """ + if not hasattr(self, 'resolved_') or self.resolved_ is None: + raise AttributeError( + "Attribute 'resolved_' is missing. You must run " + "method 'resolve()'.") + return self.resolved_ + + def resolve(self): + """ + Resolves all constraints. It adds the attribute + `resolved_`. + """ + def vars_in_values(values): + i_vals, s_vals = [], [] + for v in values: + if isinstance(v, str): + s_vals.append(v) + else: + i_vals.append(v) + return set(i_vals), s_vals + + variables = {} + for _, v in self.shapes.items(): + for sh in v.shape: + if isinstance(sh, str): + variables[sh] = None + + # first step: resolves all constraint with integer + dcsts = self.get_all_constraints() + csts = [] + for li in dcsts.values(): + csts.extend(li) + new_csts = [] + for cst in csts: + if cst.name in variables and variables[cst.name] is None: + if all(map(lambda n: isinstance(n, int), cst.values)): + variables[cst.name] = cst.values.copy() + else: + new_csts.append(cst) + else: + raise RuntimeError( # pragma: no cover + "Unable to find any correspondance for variable %r " + "in %r." % (cst.name, ", ".join(sorted(variables)))) + + # second step: everything else, like a logic algorithm + cont = True + csts = new_csts + while cont and len(new_csts) > 0: + cont = False + new_csts = [] + for cst in csts: + rvalues = variables[cst.name] + ivalues, lvars = vars_in_values(cst.values) + + if len(lvars) > 0: + miss = 0 + for lv in lvars: + if lv in variables and variables[lv] is not None: + ivalues |= variables[lv] + else: + miss += 1 + + if miss == 0: + # simple case: only integers + if rvalues is None: + inter = ivalues + else: + inter = rvalues.intersection(ivalues) + if len(inter) == 0: + raise RuntimeError( # pragma: no cover + "Resolution failed for variable %r, " + "current possibilities %r does not match " + "constraint %r." % (cst.name, rvalues, cst)) + if rvalues is None or len(inter) < len(rvalues): + variables[cst.name] = inter + else: + continue + + cont = True + new_csts.append(cst) + + # final + results = {} + for k, v in self.shapes.items(): + results[k] = v.resolve(variables) + self.resolved_ = results + return self.resolved_ diff --git a/mlprodict/onnxrt/ops_shape/shape_result.py b/mlprodict/onnxrt/ops_shape/shape_result.py index 47e600699..76cdebe22 100644 --- a/mlprodict/onnxrt/ops_shape/shape_result.py +++ b/mlprodict/onnxrt/ops_shape/shape_result.py @@ -32,6 +32,9 @@ class ShapeConstraint: def __init__(self, name, values): if name == '?': raise ValueError("Name cannot be '?'.") + if not isinstance(values, set): + raise TypeError( + "values must be a set not %r." % type(values)) self.name = name self.values = values @@ -95,7 +98,7 @@ class ShapeResult: :param shape: shape if the result is a tensor :param dtype: element type if the result is a tensor - :param sparse: is a the tensor sparse + :param sparse: is the tensor sparse :param mtype: kind of the result (see class @see cl OnnxKind) :param constraints: list of constraints applying on variables """ @@ -201,6 +204,33 @@ def merge(self, other_result): self.constraints)) return updated + def resolve(self, variables): + """ + Results variables in a shape using values stored + in *variables*. It does not copy any constraints. + + :param variables: dictionary `{ name: values }` + :return: new ShapeResult + """ + res = ShapeResult(shape=self.shape, dtype=self.dtype, + sparse=self.sparse, mtype=self.mtype) + for i in range(len(res.shape)): # pylint: disable=C0200 + v = res.shape[i] + if isinstance(v, str): + if v in variables: + vals = variables[v] + if len(vals) == 1: + res.shape[i] = list(vals)[0] + else: + raise RuntimeError( + "Unable to resolve shape %r due to ambiguities " + "for %r: %r." % (self, v, vals)) + else: + raise RuntimeError( + "Unable to resolve shape %r due to missing " + "%r." % (self, v)) + return res + @staticmethod def broadcast(sh1, sh2): """ From a9159948189dc80f7e37fc21e01c1550a6bddc03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Tue, 8 Feb 2022 18:47:09 +0100 Subject: [PATCH 08/10] fix shape inference --- _unittests/ut_onnxrt/test_shape_inference.py | 26 +++++------ mlprodict/onnxrt/onnx_shape_inference.py | 2 +- mlprodict/onnxrt/ops_shape/shape_container.py | 45 ++++++++++++++++--- mlprodict/onnxrt/ops_shape/shape_result.py | 31 +++++++++++-- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/_unittests/ut_onnxrt/test_shape_inference.py b/_unittests/ut_onnxrt/test_shape_inference.py index 4b64ae94a..0d0e332db 100644 --- a/_unittests/ut_onnxrt/test_shape_inference.py +++ b/_unittests/ut_onnxrt/test_shape_inference.py @@ -49,7 +49,7 @@ def test_shape_constraint(self): self.assertIn(sh1, shl) self.assertIn(sh2, shl) - def _test_onnx_shape_inference(self): + def test_onnx_shape_inference(self): dtype = numpy.float32 x = numpy.array([1, 2, 4, 5, 5, 4]).astype( numpy.float32).reshape((3, 2)) @@ -71,12 +71,12 @@ def _test_onnx_shape_inference(self): str(out)) self.check_infer_shapes(model_def, rt.run(), rt) cons = rt.known_shapes_.get_all_constraints() - self.assertEqual(len(cons), 2) - self.assertEqual(list(cons), ['_0', '_1']) - self.assertEqual(len(cons['_0']), 1) - cst = cons['_0'][0] - self.assertEqual(cst.name, '_0') - self.assertEqual(cst.values, {3}) + self.assertEqual(len(cons), 1) + self.assertEqual(list(cons), ['_1']) + self.assertEqual(len(cons['_1']), 1) + cst = cons['_1'][0] + self.assertEqual(cst.name, '_1') + self.assertEqual(cst.values, {'_0'}) self.assertEqual( rt.known_shapes_.names, {'_0': ('', 'X', 0), '_1': ('', 'Y', 0)}) @@ -112,20 +112,20 @@ def test_onnx_shape_inference_missing(self): self.assertEqual(len(rt.known_shapes_.names), 4) self.assertEqual(set(rt.known_shapes_.names), {'_0', '_1', '_2', '_3'}) - self.assertEqual(len(cons), 4) - self.assertEqual(list(cons), ['_0', '_1', '_2', '_3']) + self.assertEqual(len(cons), 3) + self.assertEqual(list(cons), ['_1', '_2', '_3']) self.assertEqual(len(cons['_1']), 1) cst = cons['_1'][0] self.assertEqual(cst.name, '_1') - self.assertEqual(cst.values, {2}) + self.assertEqual(cst.values, {1, 2}) self.assertEqual( rt.known_shapes_.names, {'_0': ('', 'X', 0), '_1': ('', 'X', 1), '_2': ('', 'Y', 0), '_3': ('', 'Y', 1)}) get = out.get() - self.assertEqual(get['X'].shape, get['Y'].shape) - self.assertEqual(get['Ad_C0'].shape, get['Y'].shape) - self.assertEqual(get['Ad_C0'].shape[1], 2) + self.assertEqual(get['Ad_C0'].shape, ['d0', {1, 2}]) + self.assertEqual(get['Y'].shape, ['d0', 2]) + self.assertEqual(get['X'].shape, ['d0', {1, 2}]) self.assertEqual(len(get['Ad_C0'].shape), 2) self.assertIsInstance(get['Ad_C0'].shape[0], str) diff --git a/mlprodict/onnxrt/onnx_shape_inference.py b/mlprodict/onnxrt/onnx_shape_inference.py index 570d418ba..76e3b29cb 100644 --- a/mlprodict/onnxrt/onnx_shape_inference.py +++ b/mlprodict/onnxrt/onnx_shape_inference.py @@ -91,7 +91,7 @@ def run(self, inputs=None): :param inputs: inputs :return: all results """ - known_shapes = self.known_shapes_.copy() + known_shapes = self.known_shapes_.copy(deep=True) if inputs is None: known_shapes.resolve() return known_shapes diff --git a/mlprodict/onnxrt/ops_shape/shape_container.py b/mlprodict/onnxrt/ops_shape/shape_container.py index 6dff218d0..ef3737b89 100644 --- a/mlprodict/onnxrt/ops_shape/shape_container.py +++ b/mlprodict/onnxrt/ops_shape/shape_container.py @@ -30,10 +30,10 @@ def __getitem__(self, key): "Retrieves one shape from its name." return self.shapes[key] - def copy(self): + def copy(self, deep=False): "Makes a copy." cont = ShapeContainer() - cont.shapes = self.shapes.copy() + cont.shapes = {k: v.copy(deep=deep) for k, v in self.shapes.items()} cont.names = self.names.copy() cont.names_rev = {k: v.copy() for k, v in self.names_rev.items()} return cont @@ -170,10 +170,11 @@ def vars_in_values(values): "in %r." % (cst.name, ", ".join(sorted(variables)))) # second step: everything else, like a logic algorithm - cont = True + dim_names = set() csts = new_csts - while cont and len(new_csts) > 0: - cont = False + updates = 1 + while updates > 0 and len(new_csts) > 0: + updates = 0 new_csts = [] for cst in csts: rvalues = variables[cst.name] @@ -200,11 +201,43 @@ def vars_in_values(values): "constraint %r." % (cst.name, rvalues, cst)) if rvalues is None or len(inter) < len(rvalues): variables[cst.name] = inter + updates += 1 else: continue + elif len(dim_names) > 0: + # more complex case: variables + if len(cst.values) == 1 and len(lvars) == 1: + # exact mapping between cst.name and lvars[0] + a, b = cst.name, lvars[0] + if variables[a] is None and variables[b] is not None: + if variables[b].intersection(dim_names): + variables[a] = variables[b] + updates += 1 + continue + elif variables[b] is None and variables[a] is not None: + if variables[a].intersection(dim_names): + variables[b] = variables[a] + updates += 1 + continue - cont = True new_csts.append(cst) + csts = new_csts + + if len(new_csts) > 0 and updates == 0: + # It means that a dimension needs to be left unknown. + found = None + for k, v in variables.items(): + if v is None: + found = k + if found is not None: + name = "d%d" % len(dim_names) + dim_names.add(name) + variables[found] = {name} + updates += 1 + else: + raise RuntimeError( + "Inconsistency in %r with\n%r" % ( + self, variables)) # final results = {} diff --git a/mlprodict/onnxrt/ops_shape/shape_result.py b/mlprodict/onnxrt/ops_shape/shape_result.py index 76cdebe22..569113e8e 100644 --- a/mlprodict/onnxrt/ops_shape/shape_result.py +++ b/mlprodict/onnxrt/ops_shape/shape_result.py @@ -61,6 +61,12 @@ def merge(self, cst): return self.values = self.values.intersection(cst.values) + def copy(self, deep=False): + """ + Makes a copy of the object. + """ + return ShapeConstraint(self.name, self.values.copy()) + class ShapeConstraintList: """ @@ -90,6 +96,17 @@ def __iter__(self): def __len__(self): return len(self.csts) + def copy(self, deep=False): + """ + Copies the object. + """ + cp = ShapeConstraintList() + if deep: + cp.csts = [v.copy(deep=deep) for v in self] + else: + cp.csts = self.csts.copy() + return cp + class ShapeResult: """ @@ -122,6 +139,13 @@ def __init__(self, shape=None, dtype=None, sparse=False, raise TypeError( "constraints must be of type(ShapeConstraintList).") + def copy(self, deep=False): + """ + Returns a copy for the result. + """ + return ShapeResult(self.shape, self.dtype, self.sparse, + self.mtype, self.constraints.copy(deep=deep)) + def __repr__(self): """ Usual @@ -219,12 +243,13 @@ def resolve(self, variables): if isinstance(v, str): if v in variables: vals = variables[v] + if vals is None: + raise RuntimeError( + "Inconclusive shape (None) for v=%r." % v) if len(vals) == 1: res.shape[i] = list(vals)[0] else: - raise RuntimeError( - "Unable to resolve shape %r due to ambiguities " - "for %r: %r." % (self, v, vals)) + res.shape[i] = set(vals) else: raise RuntimeError( "Unable to resolve shape %r due to missing " From 2c84ad3368a46101e0d70da40c0560695b2c9108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Tue, 8 Feb 2022 23:13:57 +0100 Subject: [PATCH 09/10] Update test_onnx_tokenizer.py --- _unittests/ut_sklapi/test_onnx_tokenizer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/_unittests/ut_sklapi/test_onnx_tokenizer.py b/_unittests/ut_sklapi/test_onnx_tokenizer.py index a3d98f754..4c931992a 100644 --- a/_unittests/ut_sklapi/test_onnx_tokenizer.py +++ b/_unittests/ut_sklapi/test_onnx_tokenizer.py @@ -8,6 +8,10 @@ import os import numpy from pyquickhelper.pycode import ExtTestCase +try: + from onnxruntime_extensions import get_library_path +except ImportError: + get_library_path = None try: from mlprodict.sklapi.onnx_tokenizer import ( SentencePieceTokenizerTransformer, GPT2TokenizerTransformer) @@ -29,6 +33,8 @@ def _load_piece(self): @unittest.skipIf(GPT2TokenizerTransformer is None, reason="onnxruntime-extensions not available") + @unittest.skipIf(get_library_path is None, + reason="onnxruntime-extensions not available") def test_sentence_piece_tokenizer_transformer(self): model, model_b64 = self._load_piece() cints = bytes(model.tolist()) @@ -64,6 +70,8 @@ def test_sentence_piece_tokenizer_transformer(self): @unittest.skipIf(GPT2TokenizerTransformer is None, reason="onnxruntime-extensions not available") + @unittest.skipIf(get_library_path is None, + reason="onnxruntime-extensions not available") def test_gpt2_tokenizer_transformer(self): vocab = os.path.join( os.path.dirname(__file__), "data", "gpt2.vocab") From f796ae481bb79ee43d0a21bf2eae731386146354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Wed, 9 Feb 2022 02:15:35 +0100 Subject: [PATCH 10/10] improves error message, add a notebook --- _doc/notebooks/loss_functions.ipynb | 833 ++++++++++++++++++++++++ _unittests/ut_npy/test_onnx_variable.py | 60 +- mlprodict/npy/__init__.py | 2 +- mlprodict/npy/onnx_numpy_wrapper.py | 11 +- mlprodict/npy/onnx_variable.py | 5 +- 5 files changed, 903 insertions(+), 8 deletions(-) create mode 100644 _doc/notebooks/loss_functions.ipynb diff --git a/_doc/notebooks/loss_functions.ipynb b/_doc/notebooks/loss_functions.ipynb new file mode 100644 index 000000000..821493960 --- /dev/null +++ b/_doc/notebooks/loss_functions.ipynb @@ -0,0 +1,833 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "95f7b5dd", + "metadata": {}, + "source": [ + "# Loss function in ONNX\n", + "\n", + "The following notebook show how to translate common loss function into ONNX." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "5d607e74", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
run previous cell, wait for 2 seconds
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from jyquickhelper import add_notebook_menu\n", + "add_notebook_menu()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ca4a486a", + "metadata": {}, + "outputs": [], + "source": [ + "from mlprodict.plotting.text_plot import onnx_simple_text_plot\n", + "%load_ext mlprodict" + ] + }, + { + "cell_type": "markdown", + "id": "4a0a7baf", + "metadata": {}, + "source": [ + "## Square loss\n", + "\n", + "The first example shows how to use [onnx](https://github.com/onnx/onnx) API to represent the square loss function $E(X,Y) = \\sum_i(x_i-y_i)^2$ where $X=(x_i)$ and $Y=(y_i)$." + ] + }, + { + "cell_type": "markdown", + "id": "9a89aaa1", + "metadata": {}, + "source": [ + "### numpy function" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0d1f4997", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.5], dtype=float32)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy\n", + "\n", + "\n", + "def square_loss(X, Y):\n", + " return numpy.sum((X - Y) ** 2, keepdims=1)\n", + "\n", + "\n", + "x = numpy.array([0, 1, 2], dtype=numpy.float32)\n", + "y = numpy.array([0.5, 1, 2.5], dtype=numpy.float32)\n", + "square_loss(x, y)" + ] + }, + { + "cell_type": "markdown", + "id": "18d432b6", + "metadata": {}, + "source": [ + "### onnx version" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6b75b7f0", + "metadata": {}, + "outputs": [], + "source": [ + "from onnx.helper import make_node, make_graph, make_model, make_tensor_value_info\n", + "from onnx import TensorProto\n", + "\n", + "nodes = [make_node('Sub', ['X', 'Y'], ['diff']),\n", + " make_node('Mul', ['diff', 'diff'], ['diff2']),\n", + " make_node('ReduceSum', ['diff2'], ['loss'])]\n", + "\n", + "graph = make_graph(nodes, 'square_loss',\n", + " [make_tensor_value_info('X', TensorProto.FLOAT, [None]),\n", + " make_tensor_value_info('Y', TensorProto.FLOAT, [None])],\n", + " [make_tensor_value_info('loss', TensorProto.FLOAT, [None])])\n", + "model = make_model(graph)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "47e630fe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "opset: domain='' version=15\n", + "input: name='X' type=dtype('float32') shape=(0,)\n", + "input: name='Y' type=dtype('float32') shape=(0,)\n", + "Sub(X, Y) -> diff\n", + " Mul(diff, diff) -> diff2\n", + " ReduceSum(diff2) -> loss\n", + "output: name='loss' type=dtype('float32') shape=(0,)\n" + ] + } + ], + "source": [ + "print(onnx_simple_text_plot(model))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "dce31928", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%onnxview model" + ] + }, + { + "cell_type": "markdown", + "id": "8acb4fe8", + "metadata": {}, + "source": [ + "Let's check it gives the same results." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0ffcf1a8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array([0.5], dtype=float32)]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from onnxruntime import InferenceSession\n", + "sess = InferenceSession(model.SerializeToString())\n", + "sess.run(None, {'X': x, 'Y': y})" + ] + }, + { + "cell_type": "markdown", + "id": "7e587692", + "metadata": {}, + "source": [ + "### second API from sklearn-onnx\n", + "\n", + "The previous API is quite verbose. [sklearn-onnx](https://onnx.ai/sklearn-onnx/) implements a more simple API to do it." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4d123a45", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "opset: domain='' version=14\n", + "input: name='X' type=dtype('float32') shape=(0,)\n", + "input: name='Y' type=dtype('float32') shape=(0,)\n", + "Sub(X, Y) -> Su_C0\n", + " Mul(Su_C0, Su_C0) -> Mu_C0\n", + " ReduceSum(Mu_C0) -> Re_reduced0\n", + "output: name='Re_reduced0' type=dtype('float32') shape=(1,)\n" + ] + } + ], + "source": [ + "from skl2onnx.algebra.onnx_ops import OnnxSub, OnnxMul, OnnxReduceSum\n", + "\n", + "diff = OnnxSub('X', 'Y')\n", + "nodes = OnnxReduceSum(OnnxMul(diff, diff))\n", + "model = nodes.to_onnx({'X': x, 'Y': y})\n", + "\n", + "print(onnx_simple_text_plot(model))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9bd6537a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array([0.5], dtype=float32)]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess = InferenceSession(model.SerializeToString())\n", + "sess.run(None, {'X': x, 'Y': y})" + ] + }, + { + "cell_type": "markdown", + "id": "e3073fb0", + "metadata": {}, + "source": [ + "As the previous example, this function only allows float32 arrays. It fails for any other type." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "1cd93361", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ONNXRuntimeError] : 2 : INVALID_ARGUMENT : Unexpected input data type. Actual: (tensor(double)) , expected: (tensor(float))\n" + ] + } + ], + "source": [ + "try:\n", + " sess.run(None, {'X': x.astype(numpy.float64), \n", + " 'Y': y.astype(numpy.float64)})\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "848a6e45", + "metadata": {}, + "source": [ + "### numpy API\n", + "\n", + "Second example is much more simple than the first one but it requires to know [ONNX operators](https://github.com/onnx/onnx/blob/main/docs/Operators.md). The most difficult type is about writing the signature. In the following example, it take two arrays of the same type `T` and returns an array of the same type, `T` being any element type (float32, float64, int64, ...)." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "80a0e035", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.5], dtype=float32)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from mlprodict.npy import onnxnumpy_np, NDArrayType\n", + "import mlprodict.npy.numpy_onnx_impl as npnx\n", + "\n", + "@onnxnumpy_np(runtime='onnxruntime',\n", + " signature=NDArrayType((\"T:all\", \"T\"), dtypes_out=('T',)))\n", + "def onnx_square_loss(X, Y):\n", + " return npnx.sum((X - Y) ** 2, keepdims=1)\n", + "\n", + "onnx_square_loss(x, y)" + ] + }, + { + "cell_type": "markdown", + "id": "fa274cae", + "metadata": {}, + "source": [ + "This API compiles an ONNX graphs for every element type. So it works float64 as well." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "b750a1ee", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.5])" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "onnx_square_loss(x.astype(numpy.float64), y.astype(numpy.float64))" + ] + }, + { + "cell_type": "markdown", + "id": "1464244f", + "metadata": {}, + "source": [ + "That's why method `to_onnx` requires to specify the element type before the method can return the associated ONNX graph." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "9cc9ab3f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "opset: domain='' version=15\n", + "input: name='X' type=dtype('float64') shape=()\n", + "input: name='Y' type=dtype('float64') shape=()\n", + "init: name='Po_Powcst' type=dtype('int64') shape=(1,) -- array([2], dtype=int64)\n", + "Sub(X, Y) -> Su_C0\n", + " Pow(Su_C0, Po_Powcst) -> Po_Z0\n", + " ReduceSum(Po_Z0, keepdims=1) -> y\n", + "output: name='y' type=dtype('float64') shape=()\n" + ] + } + ], + "source": [ + "onx = onnx_square_loss.to_onnx(key=numpy.float64)\n", + "print(onnx_simple_text_plot(onx))" + ] + }, + { + "cell_type": "markdown", + "id": "f3a6e13c", + "metadata": {}, + "source": [ + "## log loss\n", + "\n", + "The log loss is defined as the following: $L(y, s) = (1 - y)\\log(1 - p(s)) + y \\log(p(s))$ where $p(s) = sigmoid(s) = \\frac{1}{1 + \\exp(-s)}$. Let's start with the numpy version." + ] + }, + { + "cell_type": "markdown", + "id": "fe59f7e2", + "metadata": {}, + "source": [ + "### numpy function" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0d836772", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + ":5: RuntimeWarning: divide by zero encountered in log\n", + " ls = (1 - y) * numpy.log(1 - ps) + y * numpy.log(ps)\n" + ] + }, + { + "data": { + "text/plain": [ + "array([-inf], dtype=float32)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from scipy.special import expit\n", + "\n", + "def log_loss(y, s):\n", + " ps = expit(-s)\n", + " ls = (1 - y) * numpy.log(1 - ps) + y * numpy.log(ps)\n", + " return numpy.sum(ls, keepdims=1)\n", + "\n", + "y = numpy.array([0, 1, 0, 1], dtype=numpy.float32)\n", + "s = numpy.array([1e-50, 1e50, 0, 1], dtype=numpy.float32)\n", + "log_loss(y, s)" + ] + }, + { + "cell_type": "markdown", + "id": "94d04fb8", + "metadata": {}, + "source": [ + "The function may returns unexpected values because `log(0)` does not exist. The trick is usually to clip the value." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "72bc97ca", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([-16.515066], dtype=float32)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def log_loss_clipped(y, s, eps=1e-6):\n", + " ps = numpy.clip(expit(-s), eps, 1-eps)\n", + " ls = (1 - y) * numpy.log(1 - ps) + y * numpy.log(ps)\n", + " return numpy.sum(ls, keepdims=1)\n", + "\n", + "log_loss_clipped(y, s)" + ] + }, + { + "cell_type": "markdown", + "id": "48732418", + "metadata": {}, + "source": [ + "### numpy to onnx with onnx operators" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "27a13e36", + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "opset: domain='' version=15\n", + "input: name='Y' type=dtype('float32') shape=(0,)\n", + "input: name='S' type=dtype('float32') shape=(0,)\n", + "init: name='Su_Subcst' type=dtype('float32') shape=(1,) -- array([1.], dtype=float32)\n", + "init: name='Cl_Clipcst' type=dtype('float32') shape=(1,) -- array([1.e-06], dtype=float32)\n", + "init: name='Cl_Clipcst1' type=dtype('float32') shape=(1,) -- array([0.999999], dtype=float32)\n", + "Identity(Su_Subcst) -> Su_Subcst1\n", + "Sub(Su_Subcst, Y) -> Su_C0\n", + "Neg(S) -> Ne_Y0\n", + " Sigmoid(Ne_Y0) -> Si_Y0\n", + " Clip(Si_Y0, Cl_Clipcst, Cl_Clipcst1) -> Cl_output0\n", + " Sub(Su_Subcst1, Cl_output0) -> Su_C02\n", + " Log(Su_C02) -> Lo_output0\n", + " Mul(Su_C0, Lo_output0) -> Mu_C0\n", + "Log(Cl_output0) -> Lo_output02\n", + " Mul(Y, Lo_output02) -> Mu_C02\n", + " Add(Mu_C0, Mu_C02) -> Ad_C0\n", + " ReduceSum(Ad_C0, keepdims=1) -> Re_reduced0\n", + "output: name='Re_reduced0' type=dtype('float32') shape=(1,)\n" + ] + } + ], + "source": [ + "from skl2onnx.algebra.onnx_ops import (\n", + " OnnxClip, OnnxSigmoid, OnnxLog, OnnxAdd, OnnxSub, OnnxMul, OnnxNeg)\n", + "\n", + "eps = numpy.array([1e-6], dtype=numpy.float32)\n", + "one = numpy.array([1], dtype=numpy.float32)\n", + "\n", + "ps = OnnxClip(OnnxSigmoid(OnnxNeg('S')), eps, 1-eps)\n", + "ls1 = OnnxMul(OnnxSub(one, 'Y'), OnnxLog(OnnxSub(one, ps)))\n", + "ls2 = OnnxMul('Y', OnnxLog(ps))\n", + "nodes = OnnxReduceSum(OnnxAdd(ls1, ls2), keepdims=1)\n", + "model = nodes.to_onnx({'Y': y, 'S': s})\n", + "\n", + "print(onnx_simple_text_plot(model))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c4bc9615", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%onnxview model" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "7cbe7cc7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array([-16.515068], dtype=float32)]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess = InferenceSession(model.SerializeToString())\n", + "sess.run(None, {'Y': y, 'S': s})" + ] + }, + { + "cell_type": "markdown", + "id": "bc335862", + "metadata": {}, + "source": [ + "Same results." + ] + }, + { + "cell_type": "markdown", + "id": "e3c56dcc", + "metadata": {}, + "source": [ + "### numpy to onnx with numpy API" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "aaa31f99", + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([-16.515068], dtype=float32)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "@onnxnumpy_np(runtime='onnxruntime',\n", + " signature=NDArrayType((\"T:all\", \"T\"), dtypes_out=('T',)))\n", + "def onnx_log_loss(y, s, eps=1e-6):\n", + "\n", + " one = numpy.array([1], dtype=s.dtype)\n", + " ceps = numpy.array([eps], dtype=s.dtype)\n", + " \n", + " ps = npnx.clip(npnx.expit(-s), ceps, one-ceps)\n", + " ls = (one - y) * npnx.log(one - ps) + y * npnx.log(ps)\n", + " return npnx.sum(ls, keepdims=1)\n", + "\n", + "onnx_log_loss(y, s, eps=1e-6)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "b0c797bb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([-11.909897], dtype=float32)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "onnx_log_loss(y, s, eps=1e-4)" + ] + }, + { + "cell_type": "markdown", + "id": "73872dc9", + "metadata": {}, + "source": [ + "The implementation is slightly different from the numpy implementation. `1 - y` cannot be used, `numpy.array([1], dtype=s.dtype) - y` is better in this case to avoid any ambiguity on the type of constant `1`. That may be revisited in the future." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "ab735b64", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/_unittests/ut_npy/test_onnx_variable.py b/_unittests/ut_npy/test_onnx_variable.py index 1c885a843..fee83af0f 100644 --- a/_unittests/ut_npy/test_onnx_variable.py +++ b/_unittests/ut_npy/test_onnx_variable.py @@ -10,7 +10,8 @@ import mlprodict.npy.numpy_onnx_impl as nxnp from mlprodict.npy.onnx_version import FctVersion from mlprodict.npy import ( - OnnxNumpyCompiler as ONC, NDArray, NDArraySameTypeSameShape) + OnnxNumpyCompiler as ONC, NDArray, NDArraySameTypeSameShape, + NDArrayType) @ignore_warnings(DeprecationWarning) @@ -464,8 +465,65 @@ def onnx_log_1r_mul(x: NDArray[Any, numpy.float32]) -> NDArray[Any, numpy.float3 return nxnp.log(numpy.float32(2) * x) +@onnxnumpy_np(runtime='onnxruntime', + signature=NDArrayType(("T:all", "T"), dtypes_out=('T',))) +def onnx_square_loss(X, Y): + return nxnp.sum((X - Y) ** 2, keepdims=1) + + +@onnxnumpy_np(runtime='onnxruntime', + signature=NDArrayType(("T:all", "T"), dtypes_out=('T',))) +def onnx_log_loss(y, s): + one = numpy.array([1], dtype=s.dtype) + ceps = numpy.array([1e-6], dtype=s.dtype) + ps = nxnp.clip(nxnp.expit(-s), ceps, 1 - ceps) + ls = (-y + one) * nxnp.log(-ps + one) + y * nxnp.log(ps) + return nxnp.sum(ls, keepdims=1) + + +@onnxnumpy_np(runtime='onnxruntime', + signature=NDArrayType(("T:all", "T"), dtypes_out=('T',))) +def onnx_log_loss_eps(y, s, eps=1e-6): + one = numpy.array([1], dtype=s.dtype) + ceps = numpy.array([eps], dtype=s.dtype) + ps = nxnp.clip(nxnp.expit(-s), ceps, 1 - ceps) + ls = (-y + one) * nxnp.log(one - ps) + y * nxnp.log(ps) + return nxnp.sum(ls, keepdims=1) + + class TestOnnxVariable(ExtTestCase): + def test_onnx_square_loss(self): + x = numpy.array([6, 7], dtype=numpy.float32) + n1 = onnx_square_loss(x, x) + x = numpy.array([6, 7], dtype=numpy.float64) + n2 = onnx_square_loss(x, x) + self.assertEqualArray(n1, n2, decimal=4) + onx = onnx_square_loss.to_onnx(key=numpy.float32) + self.assertNotEmpty(onx) + + def test_onnx_log_loss(self): + y = numpy.array([0, 1], dtype=numpy.float32) + s = numpy.array([6, 7], dtype=numpy.float32) + n1 = onnx_log_loss(y, s) + y = y.astype(numpy.float64) + s = s.astype(numpy.float64) + n2 = onnx_log_loss(y, s) + self.assertEqualArray(n1, n2, decimal=4) + onx = onnx_log_loss.to_onnx(key=numpy.float32) + self.assertNotEmpty(onx) + + def test_onnx_log_loss_eps(self): + y = numpy.array([0, 1], dtype=numpy.float32) + s = numpy.array([6, 7], dtype=numpy.float32) + n1 = onnx_log_loss_eps(y, s) + y = y.astype(numpy.float64) + s = s.astype(numpy.float64) + n2 = onnx_log_loss_eps(y, s) + self.assertEqualArray(n1, n2, decimal=4) + onx = onnx_log_loss.to_onnx(key=numpy.float32) + self.assertNotEmpty(onx) + def test_py_abs(self): x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) y = otest_abs(x) diff --git a/mlprodict/npy/__init__.py b/mlprodict/npy/__init__.py index 6cabc1930..b1835bb86 100644 --- a/mlprodict/npy/__init__.py +++ b/mlprodict/npy/__init__.py @@ -6,7 +6,7 @@ .. versionadded:: 0.6 """ from .onnx_numpy_annotation import ( - NDArray, NDArraySameType, NDArraySameTypeSameShape, + NDArray, NDArrayType, NDArraySameType, NDArraySameTypeSameShape, Shape, DType) from .onnx_numpy_compiler import OnnxNumpyCompiler from .onnx_numpy_wrapper import onnxnumpy, onnxnumpy_default, onnxnumpy_np diff --git a/mlprodict/npy/onnx_numpy_wrapper.py b/mlprodict/npy/onnx_numpy_wrapper.py index 210b6a5a0..47d93e9f5 100644 --- a/mlprodict/npy/onnx_numpy_wrapper.py +++ b/mlprodict/npy/onnx_numpy_wrapper.py @@ -258,14 +258,17 @@ def to_onnx(self, **kwargs): return self.signed_compiled[key].compiled.onnx_ found = [] for k, v in self.signed_compiled.items(): - if k.args == key or ( - not isinstance(key, tuple) and k.args == (key, )): + if k.args == key: + found.append((k, v)) + elif isinstance(key, tuple) and k.args == key: + found.append((k, v)) + elif k.args == (key, ) * len(k.args): found.append((k, v)) if len(found) == 1: return found[0][1].compiled.onnx_ raise ValueError( - "Unable to find signature with key=%r among %r." % ( - key, list(self.signed_compiled))) + "Unable to find signature with key=%r among %r found=%r." % ( + key, list(self.signed_compiled), found)) def onnxnumpy_np(op_version=None, runtime=None, signature=None): diff --git a/mlprodict/npy/onnx_variable.py b/mlprodict/npy/onnx_variable.py index 3eb16ad7b..2009c3acf 100644 --- a/mlprodict/npy/onnx_variable.py +++ b/mlprodict/npy/onnx_variable.py @@ -56,6 +56,7 @@ class OnnxVar: .. versionadded:: 0.6 """ + __array_ufunc__ = None def __init__(self, *inputs, op=None, select_output=None, dtype=None, **kwargs): @@ -76,8 +77,8 @@ def __init__(self, *inputs, op=None, select_output=None, if (inp.size > 0 and isinstance(inp.ravel()[0], (numpy.ndarray, OnnxVar))): raise TypeError( # pragma: no cover - "Unexpected type for input %d: %r, %r." - "" % (i, type(inp), inp.ravel()[0])) + "Unexpected type for input %d: %r, %r, " + "op=%r" % (i, type(inp), inp.ravel()[0], op)) self.dtype = self._guess_dtype(dtype, from_init=True) def _guess_dtype(self, dtype, from_init=False):