diff --git a/_doc/notebooks/numpy_api_onnx.ipynb b/_doc/notebooks/numpy_api_onnx.ipynb index 4a8bbca67..5ce3ce270 100644 --- a/_doc/notebooks/numpy_api_onnx.ipynb +++ b/_doc/notebooks/numpy_api_onnx.ipynb @@ -259,7 +259,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -296,16 +296,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -333,7 +333,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "4.03 \u00b5s \u00b1 570 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "4.35 \u00b5s \u00b1 216 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -350,7 +350,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "14.2 \u00b5s \u00b1 161 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "13.1 \u00b5s \u00b1 232 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -362,7 +362,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## More complex functions with a FunctionTransformer\n", + "## Slightly more complex functions with a FunctionTransformer\n", "\n", "What about more complex functions? It is a bit more complicated too. The previous syntax does not work." ] @@ -376,7 +376,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -435,7 +435,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -469,16 +469,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -507,7 +507,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "5.75 \u00b5s \u00b1 58.7 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "5.68 \u00b5s \u00b1 66.2 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -527,7 +527,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "19.3 \u00b5s \u00b1 394 ns per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + "17.8 \u00b5s \u00b1 488 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -535,10 +535,449 @@ "%timeit custom_fct(X_train)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function transformer with FFT\n", + "\n", + "The following function is equivalent to the module of the output of a FFT transform. The matrix $M_{kn}$ is defined by $M_{kn}=(\\exp(-2i\\pi kn/N))_{kn}$. Complex features are then obtained by computing $MX$. Taking the module leads to real features: $\\sqrt{Re(MX)^2 + Im(MX)^2}$. That's what the following function does." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### numpy implementation" + ] + }, { "cell_type": "code", "execution_count": 17, "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.7378392 , 1.3990295 , 0.5351217 , 1.3990295 ],\n", + " [0.19769888, 1.1750664 , 1.7081509 , 1.1750664 ],\n", + " [0.60717905, 3.4706316 , 1.9702934 , 3.4706316 ]], dtype=float32)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def custom_fft_abs_py(x):\n", + " \"onnx fft + abs python\"\n", + " # see https://jakevdp.github.io/blog/\n", + " # 2013/08/28/understanding-the-fft/\n", + " dim = x.shape[1]\n", + " n = numpy.arange(dim)\n", + " k = n.reshape((-1, 1)).astype(numpy.float64)\n", + " kn = k * n * (-numpy.pi * 2 / dim)\n", + " kn_cos = numpy.cos(kn)\n", + " kn_sin = numpy.sin(kn)\n", + " ekn = numpy.empty((2,) + kn.shape, dtype=x.dtype)\n", + " ekn[0, :, :] = kn_cos\n", + " ekn[1, :, :] = kn_sin\n", + " res = numpy.dot(ekn, x.T)\n", + " tr = res ** 2\n", + " mod = tr[0, :, :] + tr[1, :, :]\n", + " return numpy.sqrt(mod).T\n", + "\n", + "x = numpy.random.randn(3, 4).astype(numpy.float32)\n", + "custom_fft_abs_py(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ONNX implementation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This function cannot be exported into ONNX unless it is written with ONNX operators. This is where the numpy API for ONNX helps speeding up the process." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.7378392 , 1.3990295 , 0.5351217 , 1.3990295 ],\n", + " [0.19769888, 1.1750664 , 1.7081509 , 1.1750664 ],\n", + " [0.60717905, 3.4706316 , 1.9702934 , 3.4706316 ]], dtype=float32)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from mlprodict.npy import onnxnumpy_default, onnxnumpy_np, NDArray\n", + "import mlprodict.npy.numpy_onnx_impl as nxnp\n", + "\n", + "\n", + "def _custom_fft_abs(x):\n", + " dim = x.shape[1]\n", + " n = nxnp.arange(0, dim).astype(numpy.float32)\n", + " k = n.reshape((-1, 1))\n", + " kn = (k * (n * numpy.float32(-numpy.pi * 2))) / dim.astype(numpy.float32)\n", + " kn3 = nxnp.expand_dims(kn, 0)\n", + " kn_cos = nxnp.cos(kn3)\n", + " kn_sin = nxnp.sin(kn3)\n", + " ekn = nxnp.vstack(kn_cos, kn_sin)\n", + " res = nxnp.dot(ekn, x.T)\n", + " tr = res ** 2\n", + " mod = tr[0, :, :] + tr[1, :, :]\n", + " return nxnp.sqrt(mod).T\n", + "\n", + "\n", + "@onnxnumpy_default\n", + "def custom_fft_abs(x: NDArray[Any, numpy.float32],\n", + " ) -> NDArray[Any, numpy.float32]:\n", + " \"onnx fft + abs\"\n", + " return _custom_fft_abs(x)\n", + "\n", + "\n", + "custom_fft_abs(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`custom_fft_abs` is not a function a class holding an ONNX graph. A method `__call__` executes the ONNX graph with a python runtime." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%onnxview custom_fft_abs.compiled.onnx_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Every intermediate output can be logged." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-- OnnxInference: run 39 nodes\n", + "Onnx-Shape(x) -> Sh_shape0\n", + "+kr='Sh_shape0': (2,) (dtype=int64 min=3 max=4)\n", + "Onnx-Slice(Sh_shape0, Sl_Slicecst, Sl_Slicecst1, Sl_Slicecst2) -> Sl_output01\n", + "+kr='Sl_output01': (1,) (dtype=int64 min=4 max=4)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst\n", + "+kr='Sq_Squeezecst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Squeeze(Sl_output01, Sq_Squeezecst) -> Sq_squeezed01\n", + "+kr='Sq_squeezed01': () (dtype=int64 min=4 max=4)\n", + "Onnx-Identity(Sl_Slicecst2) -> Su_Subcst\n", + "+kr='Su_Subcst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Sub(Sq_squeezed01, Su_Subcst) -> Su_C0\n", + "+kr='Su_C0': (1,) (dtype=int64 min=4 max=4)\n", + "Onnx-ConstantOfShape(Su_C0) -> Co_output01\n", + "+kr='Co_output01': (4,) (dtype=int64 min=1 max=1)\n", + "Onnx-Identity(Sl_Slicecst2) -> Cu_CumSumcst\n", + "+kr='Cu_CumSumcst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-CumSum(Co_output01, Cu_CumSumcst) -> Cu_y0\n", + "+kr='Cu_y0': (4,) (dtype=int64 min=1 max=4)\n", + "Onnx-Add(Cu_y0, Ad_Addcst) -> Ad_C01\n", + "+kr='Ad_C01': (4,) (dtype=int64 min=0 max=3)\n", + "Onnx-Cast(Ad_C01) -> Ca_output0\n", + "+kr='Ca_output0': (4,) (dtype=float32 min=0.0 max=3.0)\n", + "Onnx-Reshape(Ca_output0, Re_Reshapecst) -> Re_reshaped0\n", + "+kr='Re_reshaped0': (4, 1) (dtype=float32 min=0.0 max=3.0)\n", + "Onnx-Mul(Ca_output0, Mu_Mulcst) -> Mu_C01\n", + "+kr='Mu_C01': (4,) (dtype=float32 min=-18.84955596923828 max=-0.0)\n", + "Onnx-Mul(Re_reshaped0, Mu_C01) -> Mu_C0\n", + "+kr='Mu_C0': (4, 4) (dtype=float32 min=-56.548667907714844 max=-0.0)\n", + "Onnx-Cast(Sq_squeezed01) -> Ca_output01\n", + "+kr='Ca_output01': () (dtype=float32 min=4.0 max=4.0)\n", + "Onnx-Div(Mu_C0, Ca_output01) -> Di_C0\n", + "+kr='Di_C0': (4, 4) (dtype=float32 min=-14.137166976928711 max=-0.0)\n", + "Onnx-Identity(Sl_Slicecst2) -> Un_Unsqueezecst\n", + "+kr='Un_Unsqueezecst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Unsqueeze(Di_C0, Un_Unsqueezecst) -> Un_expanded0\n", + "+kr='Un_expanded0': (1, 4, 4) (dtype=float32 min=-14.137166976928711 max=-0.0)\n", + "Onnx-Cos(Un_expanded0) -> Co_output0\n", + "+kr='Co_output0': (1, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", + "Onnx-Sin(Un_expanded0) -> Si_output0\n", + "+kr='Si_output0': (1, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", + "Onnx-Concat(Co_output0, Si_output0) -> Co_concat_result0\n", + "+kr='Co_concat_result0': (2, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", + "Onnx-Transpose(x) -> Tr_transposed0\n", + "+kr='Tr_transposed0': (4, 3) (dtype=float32 min=-1.416614294052124 max=2.067654609680176)\n", + "Onnx-MatMul(Co_concat_result0, Tr_transposed0) -> Ma_Y0\n", + "+kr='Ma_Y0': (2, 4, 3) (dtype=float32 min=-3.7378392219543457 max=3.453752040863037)\n", + "Onnx-Pow(Ma_Y0, Po_Powcst) -> Po_Z0\n", + "+kr='Po_Z0': (2, 4, 3) (dtype=float32 min=0.0 max=13.971442222595215)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst3\n", + "+kr='Sl_Slicecst3': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst4\n", + "+kr='Sl_Slicecst4': (1,) (dtype=int64 min=1 max=1)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst5\n", + "+kr='Sl_Slicecst5': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Slice(Po_Z0, Sl_Slicecst3, Sl_Slicecst4, Sl_Slicecst5) -> Sl_output0\n", + "+kr='Sl_output0': (1, 4, 3) (dtype=float32 min=0.039084844291210175 max=13.971442222595215)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst1\n", + "+kr='Sq_Squeezecst1': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Squeeze(Sl_output0, Sq_Squeezecst1) -> Sq_squeezed0\n", + "+kr='Sq_squeezed0': (4, 3) (dtype=float32 min=0.039084844291210175 max=13.971442222595215)\n", + "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst6\n", + "+kr='Sl_Slicecst6': (1,) (dtype=int64 min=1 max=1)\n", + "Onnx-Identity(Sl_Slicecst1) -> Sl_Slicecst7\n", + "+kr='Sl_Slicecst7': (1,) (dtype=int64 min=2 max=2)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst8\n", + "+kr='Sl_Slicecst8': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Slice(Po_Z0, Sl_Slicecst6, Sl_Slicecst7, Sl_Slicecst8) -> Sl_output02\n", + "+kr='Sl_output02': (1, 4, 3) (dtype=float32 min=0.0 max=11.9284029006958)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst2\n", + "+kr='Sq_Squeezecst2': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Squeeze(Sl_output02, Sq_Squeezecst2) -> Sq_squeezed02\n", + "+kr='Sq_squeezed02': (4, 3) (dtype=float32 min=0.0 max=11.9284029006958)\n", + "Onnx-Add(Sq_squeezed0, Sq_squeezed02) -> Ad_C0\n", + "+kr='Ad_C0': (4, 3) (dtype=float32 min=0.039084844291210175 max=13.971442222595215)\n", + "Onnx-Sqrt(Ad_C0) -> Sq_Y0\n", + "+kr='Sq_Y0': (4, 3) (dtype=float32 min=0.19769887626171112 max=3.7378392219543457)\n", + "Onnx-Transpose(Sq_Y0) -> y\n", + "+kr='y': (3, 4) (dtype=float32 min=0.19769887626171112 max=3.7378392219543457)\n" + ] + }, + { + "data": { + "text/plain": [ + "array([[3.7378392 , 1.3990295 , 0.5351217 , 1.3990295 ],\n", + " [0.19769888, 1.1750664 , 1.7081509 , 1.1750664 ],\n", + " [0.60717905, 3.4706316 , 1.9702934 , 3.4706316 ]], dtype=float32)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + " custom_fft_abs(x, verbose=1, fLOG=print)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "21.3 \u00b5s \u00b1 6.81 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs_py(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "279 \u00b5s \u00b1 6.83 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using onnxruntime\n", + "\n", + "The python runtime is using numpy but is usually quite slow as the runtime needs to go through the graph structure.\n", + "*onnxruntime* is faster." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.7378392 , 1.3990295 , 0.5351217 , 1.3990295 ],\n", + " [0.19769888, 1.1750664 , 1.7081509 , 1.1750664 ],\n", + " [0.60717905, 3.4706316 , 1.9702934 , 3.4706316 ]], dtype=float32)" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "@onnxnumpy_np(runtime='onnxruntime')\n", + "def custom_fft_abs_ort(x: NDArray[Any, numpy.float32],\n", + " ) -> NDArray[Any, numpy.float32]:\n", + " \"onnx fft + abs\"\n", + " return _custom_fft_abs(x)\n", + "\n", + "\n", + "custom_fft_abs(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "72.9 \u00b5s \u00b1 659 ns per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs_ort(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inside a FunctionTransformer\n", + "\n", + "The conversion to ONNX fails if the python function is used." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FunctionTransformer is not supported unless the transform function is of type wrapped with onnxnumpy.\n" + ] + } + ], + "source": [ + "from mlprodict.onnx_conv import to_onnx\n", + "\n", + "tr = FunctionTransformer(custom_fft_abs_py)\n", + "tr.fit(x)\n", + "\n", + "try:\n", + " onnx_model = to_onnx(tr, x)\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now with the onnx version but before, the converter for FunctionTransformer needs to be overwritten to handle this functionality not available in [sklearn-onnx](https://github.com/onnx/sklearn-onnx). These version are automatically called in function [to_onnx](http://www.xavierdupre.fr/app/mlprodict/helpsphinx/mlprodict/onnx_conv/convert.html#mlprodict.onnx_conv.convert.to_onnx) from *mlprodict*." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "tr = FunctionTransformer(custom_fft_abs)\n", + "tr.fit(x)\n", + "\n", + "onnx_model = to_onnx(tr, x)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.7378392 , 1.3990295 , 0.5351217 , 1.3990295 ],\n", + " [0.19769888, 1.1750664 , 1.7081509 , 1.1750664 ],\n", + " [0.60717905, 3.4706316 , 1.9702934 , 3.4706316 ]], dtype=float32)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from mlprodict.onnxrt import OnnxInference\n", + "\n", + "oinf = OnnxInference(onnx_model)\n", + "y_onx = oinf.run({'X': x})\n", + "y_onx['variable']" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, "outputs": [], "source": [] } diff --git a/_doc/sphinxdoc/source/api/npy.rst b/_doc/sphinxdoc/source/api/npy.rst index 763104bb8..f47c5debd 100644 --- a/_doc/sphinxdoc/source/api/npy.rst +++ b/_doc/sphinxdoc/source/api/npy.rst @@ -72,6 +72,8 @@ OnnxVar .. autosignature:: mlprodict.npy.onnx_variable.OnnxVar :members: +.. _l-numpy-onnxpy-list-fct: + Available numpy functions implemented with ONNX operators +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/_doc/sphinxdoc/source/tutorial/index.rst b/_doc/sphinxdoc/source/tutorial/index.rst index 8bfe9b88a..b0336d008 100644 --- a/_doc/sphinxdoc/source/tutorial/index.rst +++ b/_doc/sphinxdoc/source/tutorial/index.rst @@ -10,5 +10,6 @@ one piece this module can do. More should follow. onnx onnx_numpy + numpy_api_onnx optim benchmark diff --git a/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst b/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst new file mode 100644 index 000000000..664dc79bc --- /dev/null +++ b/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst @@ -0,0 +1,500 @@ + +.. _l-numpy-api-for-onnx: + +Numpy API for ONNX +================== + +Many people came accross the task of converting a pipeline +including a custom preprocessing embedded into a +:epkg:`sklearn:preprocessing:FunctionTransformer`. +:epkg:`sklearn-onnx` implements many converters. Their task +is to create an ONNX graph for every :epkg:`scikit-learn` +model included in a pipeline. Every converter is a new implementation +of methods `predict`, `predict_proba` or `transform` with +:epkg:`ONNX Operators`. Every custom function is not supported. +The goal here is to make it easier for users and have their custom +function converted in ONNX. +Everybody playing with :epkg:`scikit-learn` knows :epkg:`numpy` +then it should be possible to write a function using :epkg:`numpy` +and automatically have it converted into :epkg:`ONNX`. + +.. contents:: + :local: + +Available notebooks: + +* :ref:`numpyapionnxrst` + +Principle ++++++++++ + +The user writes a function using :epkg:`numpy` function but +behind the scene, it uses an :epkg:`ONNX` runtime to execute +the function. To do that, this package reimplements many +:epkg:`numpy` functions using :epkg:`ONNX Operators`. It looks +like :epkg:`numpy` but it uses :epkg:`ONNX`. +Following example shows how to replace *numpy* by *ONNX*. + +.. runpython:: + :showcode: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + + # The numpy function + def log_1(x): + return np.log(x + 1) + + # The ONNX function + @onnxnumpy_default + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[Any, np.float32]: + return npnx.log(x + np.float32(1)) + + x = np.random.rand(2, 3).astype(np.float32) + + print('numpy') + print(log_1(x)) + + print('onnx') + print(onnx_log_1(x)) + +ONNX runtimes are usually more strict about types than :epkg:`numpy` +(see :epkg:`onnxruntime`). +By default a function must be implemented for the same input type +and there is not implicit cast. There are two important elements +in this example: + +* Decorator :func:`onnxnumpy_default `: + it parses the annotations, creates the ONNX graph and initialize a runtime with it. +* Annotation: every input and output types must be specified. They are :class:`NDArray + `, shape can be left undefined by element + type must be precised. + +`onnx_log_1` is not a function but an instance of class +:class:`wrapper_onnxnumpy `. +This class implements method `__call__` to behave like a function +and holds an attribute of type +:class:`OnnxNumpyCompiler `. +This class contains an ONNX graph and a instance of a runtime. + +* `onnx_log_1`: :class:`wrapper_onnxnumpy ` +* `onnx_log_1.compiled`: :class:`OnnxNumpyCompiler ` +* `onnx_log_1.compiled.onnx_`: ONNX graph +* `onnx_log_1.compiled.rt_fct_.rt`: runtime, by default + :class:`OnnxInference ` + +.. gdot:: + :script: DOT-SECTION + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + + # The ONNX function + @onnxnumpy_default + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[Any, np.float32]: + return npnx.log(x + np.float32(1)) + + onx = onnx_log_1.compiled.onnx_ + print(onx) + + oinf = onnx_log_1.compiled.rt_fct_.rt + print("DOT-SECTION", oinf.to_dot()) + +Available functions ++++++++++++++++++++ + +This tool does not implement every function of :epkg:`numpy`. +This a work in progress. The list of supported function is +available at :ref:`l-numpy-onnxpy-list-fct`. + +Common operators `+`, `-`, `/`, `*`, `**`, `%`, `[]` are +supported as well. They are implemented by class +:class:`OnnxVar `. +This class also implements methods such as `astype` or +properties such as `shape`, `size`, `T`. + +FunctionTransformer ++++++++++++++++++++ + +Now onnx was used to implement a custom function, +it needs to used by a :epkg:`sklearn:preprocessing:FunctionTransformer`. +One instance is added in a pipeline trained on the Iris dataset. + +.. runpython:: + :showcode: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import FunctionTransformer, StandardScaler + from sklearn.linear_model import LogisticRegression + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + from mlprodict.onnx_conv import to_onnx + from mlprodict.onnxrt import OnnxInference + + @onnxnumpy_default + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[(None, None), np.float32]: + return npnx.log(x + np.float32(1)) + + data = load_iris() + X, y = data.data.astype(np.float32), data.target + X_train, X_test, y_train, y_test = train_test_split(X, y) + + pipe = make_pipeline( + FunctionTransformer(onnx_log_1), + StandardScaler(), + LogisticRegression()) + pipe.fit(X_train, y_train) + print(pipe.predict_proba(X_test[:2])) + + onx = to_onnx(pipe, X_train[:1], rewrite_ops=True, + options={LogisticRegression: {'zipmap': False}}) + oinf = OnnxInference(onx) + print(oinf.run({'X': X_test[:2]})['probabilities']) + +*ONNX* is still more strict than *numpy*. Some elements +must be added every time this is used: + +* The custom function signature is using *float32*, + training and testing data are cast in *float32*. +* The shape of `onnx_log_1` return was changed into + `NDArray[(None, None), np.float32]`. Otherwise the converter + for *StandardScaler* raised an exception (see + :ref:`l-npy-shape-mismatch`). +* Method :func:`to_onnx ` + is called with parameter `rewrite_ops=True`. This parameter + tells the function to overwrite the converter for + *FunctionTransformer* by a new one which supports custom + functions implemented with this API (see + :ref:`l-npy-missing-converter`). + +More options +++++++++++++ + +Use onnxruntime as ONNX runtime +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, the ONNX graph is executed by the Python runtime +implemented in this module (see :ref:`l-onnx-python-runtime`). +It is a mix of :epkg:`numpy` and C++ implementations but it does +not require any new dependency. However, it is possible to use +a different one like :epkg:`onnxruntime` which has an implementation +for more :epkg:`ONNX Operators`. The only change is a wrapper +with arguments :class:`onnxnumpy_np +`: +`@onnxnumpy_np(runtime='onnxruntime')`. + +.. runpython:: + :showcode: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import FunctionTransformer, StandardScaler + from sklearn.linear_model import LogisticRegression + from onnxruntime import InferenceSession + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_np, NDArray + from mlprodict.onnx_conv import to_onnx + + @onnxnumpy_np(runtime='onnxruntime') + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[(None, None), np.float32]: + return npnx.log(x + np.float32(1)) + + data = load_iris() + X, y = data.data.astype(np.float32), data.target + X_train, X_test, y_train, y_test = train_test_split(X, y) + + pipe = make_pipeline( + FunctionTransformer(onnx_log_1), + StandardScaler(), + LogisticRegression()) + pipe.fit(X_train, y_train) + print(pipe.predict_proba(X_test[:2])) + + onx = to_onnx(pipe, X_train[:1], rewrite_ops=True, + options={LogisticRegression: {'zipmap': False}}) + + oinf = InferenceSession(onx.SerializeToString()) + print(oinf.run(None, {'X': X_test[:2]})[1]) + +Use a specific ONNX opset +^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, the ONNX graph generated by the wrapper is using +the latest version of ONNX but it is possible to use an older one +if the involved runtime does not implement the latest version. +The desired opset must be specified in two places, +the first time as an argument of `onnxnumpy_np`, the second time +as an argument of `to_onnx`. + +.. runpython:: + :showcode: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import FunctionTransformer, StandardScaler + from sklearn.linear_model import LogisticRegression + from onnxruntime import InferenceSession + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_np, NDArray + from mlprodict.onnx_conv import to_onnx + + target_opset = 11 + + @onnxnumpy_np(op_version=target_opset) + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[(None, None), np.float32]: + return npnx.log(x + np.float32(1)) + + data = load_iris() + X, y = data.data.astype(np.float32), data.target + X_train, X_test, y_train, y_test = train_test_split(X, y) + + pipe = make_pipeline( + FunctionTransformer(onnx_log_1), + StandardScaler(), + LogisticRegression()) + pipe.fit(X_train, y_train) + print(pipe.predict_proba(X_test[:2])) + + onx = to_onnx(pipe, X_train[:1], rewrite_ops=True, + options={LogisticRegression: {'zipmap': False}}, + target_opset=target_opset) + + oinf = InferenceSession(onx.SerializeToString()) + print(oinf.run(None, {'X': X_test[:2]})[1]) + +Same implementation for float32 and float64 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Only one input type is allowed by default but there is a way +to define a function supporting more than one type with +:class:`NDArrayType `. +When calling function `onnx_log_1`, input are detected and +an ONNX graph is generated and executed. Next time the same function +is called, if the input type is the same as before, it reuses the same +ONNX graph and same runtime. Otherwise, it will generate a new +ONNX graph taking this new type as input. The expression +`x.dtype` returns the type of this input in order to cast +the constant `1` into the right type before being used by +another operator. + +.. runpython:: + :showcode: + :warningout: DeprecationWarning + + import numpy as np + from onnxruntime import InferenceSession + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_np, NDArray + from mlprodict.npy.onnx_numpy_annotation import NDArrayType + from mlprodict.onnx_conv import to_onnx + + @onnxnumpy_np(signature=NDArrayType('floats'), runtime='onnxruntime') + def onnx_log_1(x): + return npnx.log(x + x.dtype(1)) + + x = np.random.rand(2, 3) + y = onnx_log_1(x.astype(np.float32)) + print(y.dtype, y) + + y = onnx_log_1(x.astype(np.float64)) + print(y.dtype, y) + +There are more options to it. Many of them are used in +:ref:`f-numpyonnxpyrt`. It is possible to add arguments +with default values or undefined number of inputs. One +important detail though, a different value for an argument +(not an input) means the ONNX graph has to be different. +Everytime input type or an argument is different, a new ONNX +graph is generated and executed. + +Common errors ++++++++++++++ + +Missing wrapper +^^^^^^^^^^^^^^^ + +The wrapper intercepts the output of the function and +returns a new function with a runtime. The inner function +returns an instance of type +:class:`OnnxVar `. +It is an layer on the top of ONNX and holds a method doing +the conversion to ONNX :meth:`to_algebra +`. + +.. runpython:: + :showcode: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[Any, np.float32]: + return npnx.log(x + np.float32(1)) + + x = np.random.rand(2, 3).astype(np.float32) + print(onnx_log_1(x)) + +Missing annotation +^^^^^^^^^^^^^^^^^^ + +The annotation is needed to determine the input and output types. +The runtime would fail executing the ONNX graph without that. + +.. runpython:: + :showcode: + :exception: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + + @onnxnumpy_default + def onnx_log_1(x): + return npnx.log(x + np.float32(1)) + +Type mismatch +^^^^^^^^^^^^^ + +As mentioned below, ONNX is strict about types. +If ONNX does an addition, it expects to do it with the same +types. If types are different, one must be cast into the other one. + +.. runpython:: + :showcode: + :exception: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + + @onnxnumpy_default + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[Any, np.float32]: + return npnx.log(x + 1) # -> replace 1 by numpy.float32(1) + + x = np.random.rand(2, 3).astype(np.float32) + print(onnx_log_1(x)) + +.. _l-npy-shape-mismatch: + +Shape mismatch +^^^^^^^^^^^^^^ + +The signature of the custom function does not specify any output shape +but the converter of the next transformer in the pipeline might +except one. + +.. runpython:: + :showcode: + :exception: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import FunctionTransformer, StandardScaler + from sklearn.linear_model import LogisticRegression + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + from mlprodict.onnx_conv import to_onnx + from mlprodict.onnxrt import OnnxInference + + @onnxnumpy_default + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[Any, np.float32]: + return npnx.log(x + np.float32(1)) + + data = load_iris() + X, y = data.data.astype(np.float32), data.target + X_train, X_test, y_train, y_test = train_test_split(X, y) + + pipe = make_pipeline( + FunctionTransformer(onnx_log_1), + StandardScaler(), + LogisticRegression()) + pipe.fit(X_train, y_train) + print(pipe.predict_proba(X_test[:2])) + + onx = to_onnx(pipe, X_train[:1], rewrite_ops=True, + options={LogisticRegression: {'zipmap': False}}) + +`NDArray[Any, np.float32]` needs to be replaced by +`NDArray[(None, None), np.float32]` to tell next converter the +output is a two dimension array. + +.. _l-npy-missing-converter: + +Missing converter +^^^^^^^^^^^^^^^^^ + +The default converter for *FunctionTransformer* implemented in +:epkg:`sklearn-onnx` does not support custom functions, +only identity, which defeats the purpose of using such preprocessing. +The conversion fails unless the default converter is replaced by +a new one supporting custom functions implemented this API. + +.. runpython:: + :showcode: + :exception: + :warningout: DeprecationWarning + + from typing import Any + import numpy as np + from sklearn.datasets import load_iris + from sklearn.model_selection import train_test_split + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import FunctionTransformer, StandardScaler + from sklearn.linear_model import LogisticRegression + import mlprodict.npy.numpy_onnx_impl as npnx + from mlprodict.npy import onnxnumpy_default, NDArray + from mlprodict.onnx_conv import to_onnx + from mlprodict.onnxrt import OnnxInference + + @onnxnumpy_default + def onnx_log_1(x: NDArray[Any, np.float32]) -> NDArray[(None, None), np.float32]: + return npnx.log(x + np.float32(1)) + + data = load_iris() + X, y = data.data.astype(np.float32), data.target + X_train, X_test, y_train, y_test = train_test_split(X, y) + + pipe = make_pipeline( + FunctionTransformer(onnx_log_1), + StandardScaler(), + LogisticRegression()) + pipe.fit(X_train, y_train) + onx = to_onnx(pipe, X_train[:1], + options={LogisticRegression: {'zipmap': False}}) + +There are a couple of ways to fix this example. One way is to call +:func:`to_onnx ` function with +argument `rewrite_ops=True`. The function restores the default +converter after the call. Another way is to call function +:func:`register_rewritten_operators ` +but changes are permanent. diff --git a/_doc/sphinxdoc/source/tutorial/onnx.rst b/_doc/sphinxdoc/source/tutorial/onnx.rst index 5eeaea3ee..f16d4a6dc 100644 --- a/_doc/sphinxdoc/source/tutorial/onnx.rst +++ b/_doc/sphinxdoc/source/tutorial/onnx.rst @@ -13,6 +13,8 @@ get the fastest python runtime but mostly to easily develop converters. .. contents:: :local: +.. _l-onnx-python-runtime: + Python Runtime for ONNX +++++++++++++++++++++++ diff --git a/_doc/sphinxdoc/source/tutorial/onnx_numpy.rst b/_doc/sphinxdoc/source/tutorial/onnx_numpy.rst index 3927851e9..3ded32846 100644 --- a/_doc/sphinxdoc/source/tutorial/onnx_numpy.rst +++ b/_doc/sphinxdoc/source/tutorial/onnx_numpy.rst @@ -196,7 +196,5 @@ This approach fixes the two issues mentioned above. The goal is write a code using the same function as :epkg:`numpy` offers but executed by an ONNX runtime. The full API is described at :ref:`l-numpy-onnxpy` and introduced here. - -**Notebooks** - -* :ref:`numpyapionnxrst` +This section is developped in notebook +:ref:`numpyapionnxrst` and :ref:`l-numpy-api-for-onnx`. diff --git a/_unittests/ut_npy/test_complex_scenario.py b/_unittests/ut_npy/test_complex_scenario.py index 14497cdc5..e6607646a 100644 --- a/_unittests/ut_npy/test_complex_scenario.py +++ b/_unittests/ut_npy/test_complex_scenario.py @@ -34,7 +34,7 @@ def custom_fft_abs_py(x): return numpy.sqrt(mod).T -def _custom_fct(x): +def _custom_fft_abs(x): dim = x.shape[1] n = nxnp.arange(0, dim).astype(numpy.float32) k = n.reshape((-1, 1)) @@ -52,15 +52,49 @@ def _custom_fct(x): @onnxnumpy_default def custom_fft_abs(x: NDArray[Any, numpy.float32], ) -> NDArray[Any, numpy.float32]: - "onnx fft" - return _custom_fct(x) + "onnx fft + abs" + return _custom_fft_abs(x) @onnxnumpy_np(runtime="onnxruntime1") def custom_fft_abs_ort(x: NDArray[Any, numpy.float32], ) -> NDArray[Any, numpy.float32]: - "onnx fft" - return _custom_fct(x) + "onnx fft + abs" + return _custom_fft_abs(x) + + +def atan2(y, x): + sx = numpy.sign(x) + sy = numpy.sign(y) + pi_part = (sy + sx * (sy ** 2 - 1)) * (sx - 1) * (-numpy.pi / 2) + atan_part = numpy.arctan(y / (x - (sx ** 2 - 1))) * sx ** 2 + return atan_part + pi_part + + +def _custom_atan2(y, x): + sx = nxnp.sign(x) + sy = nxnp.sign(y) + one = numpy.array([1], dtype=numpy.float32) + pi32 = numpy.array([-numpy.pi / 2], dtype=numpy.float32) + pi_part = (sy + sx * (sy ** 2 - one)) * (sx - one) * pi32 + atan_part = nxnp.atan(y / (x - (sx ** 2 - one))) * sx ** 2 + return atan_part + pi_part + + +@onnxnumpy_default +def custom_atan2(y: NDArray[Any, numpy.float32], + x: NDArray[Any, numpy.float32], + ) -> NDArray[Any, numpy.float32]: + "onnx atan2" + return _custom_atan2(y, x) + + +@onnxnumpy_np(runtime="onnxruntime1") +def custom_atan2_ort(y: NDArray[Any, numpy.float32], + x: NDArray[Any, numpy.float32], + ) -> NDArray[Any, numpy.float32]: + "onnx atan2" + return _custom_atan2(y, x) class TestOnnxComplexScenario(ExtTestCase): @@ -115,6 +149,23 @@ def tf_fft(x): if tfx is not None: self.assertEqualArray(tfx, fft, decimal=5) + @ignore_warnings((DeprecationWarning, RuntimeWarning)) + def test_function_transformer_atan2(self): + for rt, fct in [('py', custom_atan2), + ('ort', custom_atan2_ort)]: + with self.subTest(runtime=rt): + test_pairs = [[y, x] for x in [3., -4., 0.] + for y in [5., -6., 0.]] + y_val = numpy.array( + [y for y, x in test_pairs], dtype=numpy.float32) + x_val = numpy.array( + [x for y, x in test_pairs], dtype=numpy.float32) + exp = atan2(y_val, x_val) + self.assertEqualArray( + numpy.arctan2(y_val, x_val), exp, decimal=5) + got = fct(y_val, x_val) + self.assertEqualArray(exp, got, decimal=5) + if __name__ == "__main__": unittest.main() diff --git a/_unittests/ut_npy/test_onnx_variable.py b/_unittests/ut_npy/test_onnx_variable.py index 75d22fc38..fa176669d 100644 --- a/_unittests/ut_npy/test_onnx_variable.py +++ b/_unittests/ut_npy/test_onnx_variable.py @@ -288,6 +288,12 @@ def test_abs_log_multi(x): return nxnp.log(nxnp.abs(x)) +@onnxnumpy_np(signature=NDArraySameTypeSameShape("floats")) +def test_abs_log_multi_dtype(x): + "onnx numpy log multiple type" + return nxnp.log(nxnp.abs(x) + x.dtype(1)) + + @onnxnumpy_default def test_abs_shape(x: NDArray[Any, numpy.float32], ) -> NDArray[Any, numpy.int64]: @@ -526,6 +532,11 @@ def test_py_abs_log_multi(self): y = test_abs_log_multi(x) self.assertEqualArray(y, numpy.log(numpy.abs(x))) + def test_py_abs_log_multi_dtype(self): + x = numpy.array([[6.1, -5], [-3.5, 7.8]], dtype=numpy.float32) + y = test_abs_log_multi_dtype(x) + self.assertEqualArray(y, numpy.log(numpy.abs(x) + 1)) + def test_py_abs_shape(self): x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) y = test_abs_shape(x) diff --git a/mlprodict/npy/onnx_numpy_compiler.py b/mlprodict/npy/onnx_numpy_compiler.py index 1f869d379..21ca0abd4 100644 --- a/mlprodict/npy/onnx_numpy_compiler.py +++ b/mlprodict/npy/onnx_numpy_compiler.py @@ -7,6 +7,7 @@ import inspect from typing import Any import numpy +from skl2onnx.common.data_types import guess_numpy_type from ..onnxrt import OnnxInference from .onnx_numpy_annotation import get_args_kwargs from .onnx_variable import OnnxVar @@ -242,7 +243,7 @@ def _possible_names(): if a == "op_version": continue if a not in annotations: - raise RuntimeError( # pragma: no cover + raise RuntimeError( "Unable to find annotation for argument %r. " "You should annotate the arguments and the results " "or specify a signature." % a) @@ -281,7 +282,8 @@ def _to_onnx(self, op_version=None, signature=None, version=None): getattr(self.fct_, '__module__', None))) names_in = [oi[0] for oi in inputs] names_out = [oi[0] for oi in outputs] - names_var = [OnnxVar(n) for n in names_in] + names_var = [OnnxVar(n, dtype=guess_numpy_type(dt[1])) + for n, dt in zip(names_in, inputs)] if 'op_version' in self.fct_.__code__.co_varnames: onx_algebra = self.fct_( *names_in, op_version=op_version, **kwargs) diff --git a/mlprodict/npy/onnx_variable.py b/mlprodict/npy/onnx_variable.py index d62535d47..faa87730f 100644 --- a/mlprodict/npy/onnx_variable.py +++ b/mlprodict/npy/onnx_variable.py @@ -45,17 +45,24 @@ class OnnxVar: :param select_output: if multiple output are returned by ONNX operator *op*, it takes only one specifed by this argument + :param dtype: specifies the type of the variable + held by this class (*op* is None) in that case :param kwargs: addition argument to give operator *op* .. versionadded:: 0.6 """ - def __init__(self, *inputs, op=None, select_output=None, **kwargs): + def __init__(self, *inputs, op=None, select_output=None, + dtype=None, **kwargs): self.inputs = inputs self.select_output = select_output self.onnx_op = op self.alg_ = None self.onnx_op_kwargs = kwargs + self.dtype = dtype + if dtype is not None and (op is not None or len(inputs) != 1): + raise RuntimeError( + "dtype can only be used if op is None or len(inputs) == 1.") for i, inp in enumerate(self.inputs): if isinstance(inp, type): raise TypeError( @@ -69,7 +76,7 @@ def to_algebra(self, op_version=None): if self.onnx_op is None: if len(self.inputs) != 1: raise RuntimeError( # pragma: no cover - "Unexpected numer of inputs, 1 expected, " + "Unexpected number of inputs, 1 expected, " "got {} instead.".format(self.inputs)) self.alg_ = self.inputs[0] else: