Use numpy API for ONNX to write custom converters

_doc/sphinxdoc/source/api/npy.rst

@@ -53,19 +53,35 @@ as opposed to numpy. This approach is similar to what
 :epkg:`tensorflow` with `autograph
 <https://www.tensorflow.org/api_docs/python/tf/autograph>`_.
 
-NDArray
-+++++++
+Signatures
+++++++++++
 
 .. autosignature:: mlprodict.npy.onnx_numpy_annotation.NDArray
     :members:
 
-onnxnumpy
-+++++++++
+.. autosignature:: mlprodict.npy.onnx_numpy_annotation.NDArraySameType
+    :members:
+
+.. autosignature:: mlprodict.npy.onnx_numpy_annotation.NDArraySameTypeSameShape
+    :members:
+
+.. autosignature:: mlprodict.npy.onnx_numpy_annotation.NDArrayType
+    :members:
+
+.. autosignature:: mlprodict.npy.onnx_numpy_annotation.NDArrayTypeSameShape
+    :members:
+
+Decorators
+++++++++++
 
 .. autosignature:: mlprodict.npy.onnx_numpy_wrapper.onnxnumpy
 
 .. autosignature:: mlprodict.npy.onnx_numpy_wrapper.onnxnumpy_default
 
+.. autosignature:: mlprodict.npy.onnx_numpy_wrapper.onnxnumpy_np
+
+.. autosignature:: mlprodict.npy.onnx_sklearn_wrapper.onnxsklearn_transformer
+
 OnnxNumpyCompiler
 +++++++++++++++++
 
@@ -78,6 +94,11 @@ OnnxVar
 .. autosignature:: mlprodict.npy.onnx_variable.OnnxVar
     :members:
 
+Registration
+++++++++++++
+
+.. autosignature:: mlprodict.npy.onnx_sklearn_wrapper.update_registered_converter_npy
+
 .. _l-numpy-onnxpy-list-fct:
 
 Available numpy functions implemented with ONNX operators

_unittests/ut_npy/test_complex_scenario.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 """
-@brief      test log(time=3s)
+@brief      test log(time=21s)
 """
 import unittest
 import warnings

_unittests/ut_npy/test_custom_transformer.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+"""
+@brief      test log(time=3s)
+"""
+import unittest
+import warnings
+from logging import getLogger
+import numpy
+from sklearn.base import TransformerMixin, BaseEstimator
+from sklearn.decomposition import PCA
+from pyquickhelper.pycode import ExtTestCase, ignore_warnings
+from skl2onnx import update_registered_converter
+from skl2onnx.algebra.onnx_ops import (  # pylint: disable=E0611
+    OnnxIdentity, OnnxMatMul, OnnxSub)
+from skl2onnx.algebra.onnx_operator import OnnxSubOperator
+from skl2onnx.common.data_types import guess_numpy_type
+from mlprodict.onnx_conv import to_onnx
+from mlprodict.onnxrt import OnnxInference
+from mlprodict.npy import onnxsklearn_transformer
+
+
+class DecorrelateTransformer(TransformerMixin, BaseEstimator):
+    def __init__(self, alpha=0.):
+        BaseEstimator.__init__(self)
+        TransformerMixin.__init__(self)
+        self.alpha = alpha
+
+    def fit(self, X, y=None, sample_weights=None):
+        self.pca_ = PCA(X.shape[1])  # pylint: disable=W0201
+        self.pca_.fit(X)
+        return self
+
+    def transform(self, X):
+        return self.pca_.transform(X)
+
+
+def decorrelate_transformer_shape_calculator(operator):
+    op = operator.raw_operator
+    input_type = operator.inputs[0].type.__class__
+    input_dim = operator.inputs[0].type.shape[0]
+    output_type = input_type([input_dim, op.pca_.components_.shape[1]])
+    operator.outputs[0].type = output_type
+
+
+def decorrelate_transformer_converter(scope, operator, container):
+    op = operator.raw_operator
+    opv = container.target_opset
+    out = operator.outputs
+    X = operator.inputs[0]
+    subop = OnnxSubOperator(op.pca_, X, op_version=opv)
+    Y = OnnxIdentity(subop, op_version=opv, output_names=out[:1])
+    Y.add_to(scope, container)
+
+
+class DecorrelateTransformer2(DecorrelateTransformer):
+    pass
+
+
+def decorrelate_transformer_converter2(scope, operator, container):
+    op = operator.raw_operator
+    opv = container.target_opset
+    out = operator.outputs
+    X = operator.inputs[0]
+    dtype = guess_numpy_type(X.type)
+    m = OnnxMatMul(
+        OnnxSub(X, op.pca_.mean_.astype(dtype), op_version=opv),
+        op.pca_.components_.T.astype(dtype), op_version=opv)
+    Y = OnnxIdentity(m, op_version=opv, output_names=out[:1])
+    Y.add_to(scope, container)
+
+
+class DecorrelateTransformer3(DecorrelateTransformer):
+    pass
+
+
+@onnxsklearn_transformer(register_class=DecorrelateTransformer3)
+def decorrelate_transformer_converter3(X, op=None):
+    if X.dtype is None:
+        raise AssertionError("X.dtype cannot be None.")
+    mean = op.pca_.mean_.astype(X.dtype)
+    cmp = op.pca_.components_.T.astype(X.dtype)
+    return (X - mean) @ cmp
+
+
+class TestCustomTransformer(ExtTestCase):
+
+    def setUp(self):
+        logger = getLogger('skl2onnx')
+        logger.disabled = True
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ResourceWarning)
+            update_registered_converter(
+                DecorrelateTransformer, "SklearnDecorrelateTransformer",
+                decorrelate_transformer_shape_calculator,
+                decorrelate_transformer_converter)
+            update_registered_converter(
+                DecorrelateTransformer2, "SklearnDecorrelateTransformer2",
+                decorrelate_transformer_shape_calculator,
+                decorrelate_transformer_converter2)
+
+    @ignore_warnings((DeprecationWarning, RuntimeWarning))
+    def test_function_transformer(self):
+        X = numpy.random.randn(20, 2).astype(numpy.float32)
+        dec = DecorrelateTransformer()
+        dec.fit(X)
+        onx = to_onnx(dec, X.astype(numpy.float32))
+        oinf = OnnxInference(onx)
+        exp = dec.transform(X)
+        got = oinf.run({'X': X})
+        self.assertEqualArray(exp, got['variable'])
+
+    @ignore_warnings((DeprecationWarning, RuntimeWarning))
+    def test_function_transformer2(self):
+        X = numpy.random.randn(20, 2).astype(numpy.float32)
+        dec = DecorrelateTransformer2()
+        dec.fit(X)
+        onx = to_onnx(dec, X.astype(numpy.float32))
+        oinf = OnnxInference(onx)
+        exp = dec.transform(X)
+        got = oinf.run({'X': X})
+        self.assertEqualArray(exp, got['variable'])
+
+    @ignore_warnings((DeprecationWarning, RuntimeWarning))
+    def test_function_transformer3_float32(self):
+        X = numpy.random.randn(20, 2).astype(numpy.float32)
+        dec = DecorrelateTransformer3()
+        dec.fit(X)
+        onx = to_onnx(dec, X.astype(numpy.float32))
+        oinf = OnnxInference(onx)
+        exp = dec.transform(X)
+        got = oinf.run({'X': X})
+        self.assertEqualArray(exp, got['variable'])
+        X2 = decorrelate_transformer_converter3(X, op=dec)
+        self.assertEqualArray(X2, got['variable'])
+
+
+if __name__ == "__main__":
+    unittest.main()

_unittests/ut_npy/test_onnxpy.py

@@ -8,6 +8,7 @@
 from onnxruntime.capi.onnxruntime_pybind11_state import InvalidArgument  # pylint: disable=E0611
 from pyquickhelper.pycode import ExtTestCase
 from mlprodict.npy import OnnxNumpyCompiler as ONC, NDArray
+from mlprodict.npy.onnx_variable import OnnxVar
 from mlprodict.npy.onnx_numpy_annotation import _NDArrayAlias
 from skl2onnx.algebra.onnx_ops import OnnxAbs  # pylint: disable=E0611
 from skl2onnx.common.data_types import FloatTensorType
@@ -25,6 +26,17 @@ def onnx_abs_shape(x: NDArray[(Any, Any), numpy.float32],
                        op_version=None) -> NDArray[(Any, Any), numpy.float32]:
         return OnnxAbs(x, op_version=op_version)
 
+    def test_onnx_var(self):
+        ov = OnnxVar('X')
+        rp = repr(ov)
+        self.assertEqual("OnnxVar('X')", rp)
+        ov = OnnxVar('X', op=OnnxAbs)
+        rp = repr(ov)
+        self.assertEqual("OnnxVar('X', op=OnnxAbs)", rp)
+        ov = OnnxVar('X', op='filter')
+        rp = repr(ov)
+        self.assertEqual("OnnxVar('X', op='filter')", rp)
+
     def test_process_dtype(self):
         for name in ['all', 'int', 'ints', 'floats', 'T']:
             res = _NDArrayAlias._process_type(  # pylint: disable=W0212

_unittests/ut_onnxrt/test_onnxrt_runtime_empty.py

@@ -50,7 +50,8 @@ def test_onnxt_runtime_empty_unknown(self):
         Z = helper.make_tensor_value_info(
             'Z', TensorProto.FLOAT, [None, 2])  # pylint: disable=E1101
         node_def = helper.make_node('Add', ['X', 'Y'], ['Zt'], name='Zt')
-        node_def2 = helper.make_node('AddUnknown', ['X', 'Zt'], ['Z'], name='Z')
+        node_def2 = helper.make_node(
+            'AddUnknown', ['X', 'Zt'], ['Z'], name='Z')
         graph_def = helper.make_graph(
             [node_def, node_def2], 'test-model', [X, Y], [Z])
         model_def = helper.make_model(graph_def, producer_name='onnx-example')

mlprodict/npy/__init__.py

@@ -10,3 +10,5 @@
     Shape, DType)
 from .onnx_numpy_compiler import OnnxNumpyCompiler
 from .onnx_numpy_wrapper import onnxnumpy, onnxnumpy_default, onnxnumpy_np
+from .onnx_sklearn_wrapper import (
+    onnxsklearn_transformer, update_registered_converter_npy)