From 5aa5bc9b10bdc8b4cbf29b2459d88ad8b444b83f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 18 Mar 2021 13:52:49 +0100 Subject: [PATCH 01/10] Complete tutorial on numpy API for ONNX --- _doc/notebooks/numpy_api_onnx.ipynb | 904 +++---------- _doc/notebooks/numpy_api_onnx2.ipynb | 1115 +++++++++++++++++ _doc/sphinxdoc/source/api/npy.rst | 2 + .../source/tutorial/numpy_api_onnx.rst | 4 +- _unittests/ut_tools/test_zoo.py | 18 +- 5 files changed, 1311 insertions(+), 732 deletions(-) create mode 100644 _doc/notebooks/numpy_api_onnx2.ipynb diff --git a/_doc/notebooks/numpy_api_onnx.ipynb b/_doc/notebooks/numpy_api_onnx.ipynb index 776b3ff40..af7509b3f 100644 --- a/_doc/notebooks/numpy_api_onnx.ipynb +++ b/_doc/notebooks/numpy_api_onnx.ipynb @@ -4,9 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Introduction to a numpy API for ONNX\n", + "# Introduction to a numpy API for ONNX: CustomClassifier\n", "\n", - "This notebook shows how to write python functions similar functions as numpy offers and get a function which can be converted into ONNX." + "This notebook shows how to write python classifier using similar functions as numpy offers and get a class which can be inserted into a pipeline and still be converted into ONNX." ] }, { @@ -165,876 +165,357 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## A pipeline with FunctionTransformer" + "## A custom binary classifier\n", + "\n", + "Let's imagine a classifier not that simple about simple but not that complex about predictions. It does the following:\n", + "* compute the barycenters of both classes,\n", + "* determine an hyperplan containing the two barycenters of the clusters,\n", + "* train a logistic regression on both sides.\n", + "\n", + "Some data first..." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.datasets import load_iris\n", - "from sklearn.model_selection import train_test_split\n", - "data = load_iris()\n", - "X, y = data.data, data.target\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, "outputs": [ { "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAm7klEQVR4nO3dfZAc5Z0f8O9Pq92dud2V4GD8EgS7cKLupLgoBCvjFLgSGUF8LseAzgb2rghYy4GrYmLIlQ9fhLk/xFWdC4wtjHNrcsshX7wLhXEcx3EsW4YKduXIaoV0CUj2nQISXoI9gwxCEnrZ1f7yx7Mt9fZ0z3T39Ht/P1VT0s7Oy9M9s8+vn7ffI6oKIiIqnyVpF4CIiNLBAEBEVFIMAEREJcUAQERUUgwAREQltTTtAgRx7rnn6tDQUNrFICLKlZ07d76pqjXn/bkKAENDQ5ienk67GEREuSIiB9zuZxcQEVFJMQAQEZUUAwARUUnlagyAiKgTs7OzmJmZwfHjx9MuSiwqlQpWrFiB7u5uX49nACCi0piZmcHAwACGhoYgImkXJ1KqioMHD2JmZgYXXnihr+ewC4gobxoNYMcO8y8Fcvz4cZxzzjmFq/wBQERwzjnnBGrdMAAQ5cnkJDA4CFxzjfl3cjLtEuVOESt/S9BjYwAgyotGAxgdBY4dAw4dMv+OjrIlQKExABDlxf79QE/P4vu6u839VGivvvoqrrjiCqxcuRI33XQTTp48GcnrMgAQ5cXQEOD8w5+dNfdTod1777245557sG/fPpx99tkYHx+P5HUZAIjyolYDxseBahVYtsz8Oz5u7qfYRDnmfv/99+OrX/3q6Z83bdqELVu2tHyOquLZZ5/FJz/5SQDArbfeiu9+97udFwacBkqULyMjwPr1pttnaIiVf8wmJ80wS0+PaXyNj5uPIKyNGzdiw4YNuPvuuzE/P48nn3wSzz77LC699FLXx09MTOA973kPzjrrLCxdaqrrFStW4PXXXw9fCBsGAKK8qdVY8SfAPuZ+7Ji5b3TUxN+wp39oaAjnnHMOdu3ahV//+tdYs2YNBgcHsXv3bs/nvPnmm+HezAcGACIiF9aYu1X5A2fG3DuJv7fffjueeOIJ/OpXv8LGjRtx+PBhfPjDH3Z97MTEBFatWoW3334bc3NzWLp0KWZmZnDeeeeFL4ANAwARkYu4xtxvuOEG3H///ZidncXExAS6urpatgAAYN26dfj2t7+Nm2++GVu3bsV1113XWSEWcBCYiMhFXGPuPT09WLduHW688UZ0dXX5es6XvvQlPPzww1i5ciUOHjyI0dHRzgqxgC0AIiIPcYy5z8/P44UXXsDTTz/t+zkXXXQRpqamOn9zB7YAiIhaqNWAtWujqfz37NmDlStX4uqrr8bFF1/c+Qt2iC0AIqKErF69Gq+88kraxTiNLQAiopJiACAiKikGACKikmIAICIqKQYAIqKMe/TRR7Fy5UqISKSpIRgAiIgy7sorr8T27dsxODgY6esyABARtRJhPugw6aABYM2aNRiKYd8HrgMgIvIScT7oMOmgV69eHfr92mEAICJyE0M+6DDpoOOUWgAQkfMBfBPAewEogMdUtX1biIgoCTHlgw6aDrqoLYA5AH+iqi+KyACAnSLyY1Xdk2KZiIiMmPJBh0kHHZfUBoFV9Q1VfXHh/4cB7AUQzS4HRESdiikfdJh00I888ghWrFiBmZkZXHLJJbj99ts7KoNFVDWSF+qoECJDAJ4H8AFVfcfxuzsA3AEAF1xwweUHDhxIvoBEVAh79+7FqlWrgj2p0Yg0H/T8/Dwuu+wyPP3007FkBHU7RhHZqarDzsemPg1URPoBPAPgbmflDwCq+piqDqvqcI37oBLlU4RTKRMXYT5opoO2EZFumMr/W6r6nTTLQkQxiXgqZZ4xHfQCEREA4wD2qurDaZWDiGJkn0p56JD5d3Q01ZZAFrq94xL02NLsAroSwC0APiIiuxduH0uxPFQmee6SyBNrKqWdNZUyBZVKBQcPHixkEFBVHDx4EJVKxfdzUusCUtWfAZC03p9KjF0SyYlpKmVY1kyaRkEDf6VSwYoVK3w/PhOzgPwaHh7W6enptItBedZoAIODixf3VKvAgQPRbPpKzayA291tKn8G3MR5zQJiKggql5hWd1ILIyMmfUKEUykpGgwAVC4Z65IojVqNFX8Gpb4OgChRMa3uJMojtgCofNglQQSAAYDKil0SROwCIiIqKwYAIqKSYgAgIiopBgCioJhGggqCAYAoiMlJs5L4mmvMv5OTybwvgw7FgAGAyK+0MlumFXSo8BgAiPxKI7NlBtMpU3EwABD5lUYaiYylU6ZiYQAg8iuNNBLMXUQxYgAgCmJkxKSO3r7d/Bt3WmPmLqIYMRUEUVBJp5Fg7iKKCQNAWTUarFDyhLmLKAbsAiojTiukPOJaiMgxAJSN27TCT38a2Ls37ZLlAyuhdPCiJRYMAGXjNq3wxAlgzRr+UbXjVgkxIMSPayFiwwBQJo0G8NZbpsJ3OnGCf1StuFVCt97Kq9IkcC1EbBgAysK6er3xRmB+Hujqan4M/6i8uVVCs7O8Kk0C10LEhgGgDJxXrydPmsrerULjH5W7oSHg3XdbP4YBNB5cCxEbBoAycLt67ekBvvhF/lEFodr6950GUI4nePNYgMdT1hkGgDLwakLfeWeyq1rzbNcuYG6u+f7e3mgCaEyzXApVQdZqwNq1p88xJwZ1jgGgDFo1oR1/VBTQN7/ZeQCNaZZLkStITgyKBlcClwXTCXRmzRrTxz87e+a+7m5g3brOz6XVRXfs2OLX3r8/9GvbK0jrZUdHzVegCB99DKeslNgCKBNe7YdXqwFbtwKVCtDXZ/7dujWacxnDLBe3YZ/3dTXQ+EEx+oM4MSgaDACUL2l2ao+MAK+9Bjz3nPk3qjGTGGa5OCvImzGJl44M4vfuKkZ/ECcGRUO03cyGDBkeHtbp6em0i0FJsxLXvfgicM895tL25EnzF1+kgeuIE/RNTppun/d1NfDSkUH8Fmz9JdWqGbfIeY3JnIb+iMhOVR1uup8BoOSy/hdk1WJLlwKHDy/+XUEqsTg1GkDjBzvwe3ddgyWHD535xbJlZvB67dr0CkeJ8QoA7AIqs6xPE7GPZDorf4ALr3yo1YDVHxvCkjl2mFMzBoCyimoeXZx98m4jmXasxPxhhzl5YAAoqygSbMXdgnCb6gEAAwOsxIJKeitLygWOAZRVo2Eq7WMhBwY7fb5f1hiANQf/K18BLrss+JhF1sc6iGLEMQBarNNugaRS9DqvXO+8M/hahqyPdRClhC2Asgt7ZZxUC6BTeSknUYwy2QIQkcdFpC4iL6VZjlILuzo4LwOL3EyEyFPauYCeAPAogG+mXA4KIw/5hZgzgMhTqi0AVX0ewG/SLAN1KO78Qp1OMw3bUilUHuVs4ilOX+YHgUXkDhGZFpHpBr8p5eIcvH3ggXC1RdApkBw0jh1PcTakPggsIkMAvq+qH2j3WA4Cl4jb4C1gsnA+/nh889izNmhcwOmrWTvFZZDJQWAiT16rgI8fj3fnjywNGtsvky+4IHwLKCOsLp9du5pP8bFjwDe+kU65yowBgLLJaxUw4F4hR9WhnJVBY2eqjuPHzR7OOe0vsWLZ1VcDn/gEcPRo82NyHt9yKe1poJMA/g7A74rIjIiMplkeyhD74K2Ts0KOskM5K9NbvVpACex9GPXgrDOn34kT7tsrnzjBVkDiVDU3t8svv1ypZOp11c2bVSsV1WXLVKtV1YmJxb+vVlWBM7dq1dzf6ftOTXX+Op28v/O4rNuyZaZsMZiYMG+7fHnzqQ5rasp8fG6H4rxF8dFRMwDT6lKnsguIsq1WA+67z+zA5TaLx6vPfteuzqePprl9ptUSqVSafxdTl1RcG63395seLCe3Bg7X6CWLAYDywatCHhoC3n138X1HjgDXX99Zl1AWJqlbW1Bu3pxIl1Rc499HjrjHMZHm+7lGL1kMAJR/Iot/np/v7DLWa0whjaBgtYASSOUc1/j30FDzRwQAW7aYGb1pD7eUWdqpIIg6s3+/qTm8ZgwBZy5j/aa5tvpBrInqo6PAO++E2484qnn8tVrsNaPV62TPvu130XSrQ7S/bleXed0tW0xiVyD72USKLPWFYEFwIViHCrioyHPBmF2QVUY7dpgr/0O2/XP7+02tdeJEsNe09jKwBY3G+pHMfwReXxO3+10O0TMuFvHrlxdeC8FSn9kT5FbqWUCdzkqJY3pHFthnCQ0MuE8tGRsL9nrO2Te9vc2vPTCg+sQT3p+Hy+vM9lT1/Eo9lx+B29cnrglYFD1wFlBONRpmhcwFF4Qf1Ixjekcc/eFBX9Pqq3/oIdPJ/Ed/ZLaLtOvvNzuI+eW2DmDLluaJ64cPA3fd5f15uIyovntyKd57fH+kM2yS4PX1cVvRy1k8OeMWFbJ6K10LwLrs6nSy9NSUuXSLai55HK2JoK/pcvk5X6k2TzgPe0nqbHFZ5XNrZbi9h1v5AL0bDyYxnT869bq+/MSUXjRQb/r6bNvGFkBewKMFkHqlHuRWqgAQ5UKgKNvqcbT7w7ymS1B7G8t096c2m+e6LRrrVL1uun2cQcDr8xgba/rs5gG9HWMKmFi1bZvLYaa9CM2yEPRODSzXo6jqTZho+nisuBjH6aboMADkjdtVeycVblR/qVG3JsK+Zr2u846gcRSmj72xJ+IK1F4hBwlWU1OqfX1Nn98x9Or7l9a1p8elwZOVsRqX4zyKql7YX3ddjJ2FeEXeGADypF53b19bl41hK4Uo/lKz0gJQ1X2bJ/Qoqvo2lp2+Qr2w33RZRFYbuVXIfoNpvW4GkB2f4clqv17VO9V0uI09GRpVdQnKpwaW6ctPTPkrju27xgCRPgaAvLBXON3dqj09ZyqazZuz8VcUR7s/xGs29tT14z3bdD226bmo680wAeHUwPJoytUqMPmt1Vy6geZ6q6596i8/EUPrKqxOAr3tOzzbU9VbuidSb9CUHQNAHrj90Xl2FKcsjsu6IK+5UMmcqJr+6bt6x/QoIr56jqq7a2zMtAT6+1WrVT00NuH6MT/7VHO3VmSD2MF+bYQJ9B5dR+eizkHiFDEA5EEc/etF5FLJnOrp1VN9PgdnO3ifqCpke91qNfSWL1e9pXtCZ3s6bF21GUcINMwQNNB7DM4PY4pf5xQxAORBmVbWdNKC2LateXB1YKC5vz2KcxfjNBevoZ7zK3X9zbaQ58Zt+mm1qi9uq5/uufI1Uzbs58MWQCZ5BQAuBMuSLGxGkkTCsyAbuDjLMzkJXHdd85ZSc3NmwVbU5y7ohvIB1GrA2Wc3L6Y61FPDvrNDpqJ2WYD2zrFu/LsN+zE4aBZwOVMzNy3e6mSDHcd3eK6nis90j+Pkslr8X+csZHDNG7eokNVb4VsAlrSmTSQxBTFIK8dZnrEx74VxVllzNuUk8kZfmytwt1ulYnu/qAqU9CygrEyfzSiwC4haSqr7ye84h9+cPH19ph8lA8JWdJH3Mk2YcQT79Fivyh8wk8tO63T6ZxrK1HUaklcAYBcQGXHtBuLkN+m8V3mcz52fB9asibaMIXTSa+LZyxSyS6OxfgQXLTmA9diOQRzAU/DuturtPZOWGYDr53P88Cz+1V1D2d2PPqnvbgExACQtq/2Uce0G4mT1Edv/YOfmTO3XrjynTgXr50/oXEeRa69pw7MOIsr+/cA7vTVMYy3eROsO9y1bHKfP1oc/P7AM76KKjRjHK4dr2U1gl9R3t4jcmgXWDcAyAL/jcv8lrZ4X1y33XUBZ76dMKrGL36koXuXx09eS4LmOfPauV5fGnj2++pi8es8efNDc399vfm6VJbuxp67/9f4pHeprXrCWyWmcTErUEoKOAQC4EcD/A7AbwMsA1tp+96LX8+K85ToA5KWfMokRuyA1ZpjyJHyuI387t/NTrZpa22dAc9aHz4yZ89jYU/cdO/0mPs2MnE0ASFKYALAbwPsX/v9BAD8HcMPCz7u8nhfnLdcBgIu8zoi7gk7hXEd6AdoqE2yA82XVh798cELnetunyLDWJTgbZ9YyC15Y55dXAGg1BtClqm8sdBNNAVgH4D4R+bcANLI+qLJgP+UZca93SOFcR7pcwHl+envN/+18DnL+9DsN/PbnR9F14hiWHPYeoLCGHDZsaF4n0N8PfO1rse5HT2lxiwomYOB/wtH/D2AAwE8AnPB6Xpy3XLcAVNlP6RSgyR64dR/FuU67S8F6/z17AreYJibMlfwwpvQtLF/03FMDi1tD7Rocme72IV8QogvoEgArXe7vBnC/1/PivOU+AKimX6kkJcLjDD2e20kZsjZgHyCg2Sv0c1FvSpI317u4RvfaeqKvr3226zJ8lYsgTAB4BcCfwnQFWfe9F8B/8nqxuG+FCABlEGHlmcrYeVYH7H3WuM4K/SYs3jfh0Njiz8PrcFsloc1afKTWvOrsVmMAlwO4CMBuEfmIiHwOwBSAv4MZFCZqFvEG9Kms8cnqwiL7YoEWaxycQyBPYQSDOICP927HD8cOYNmdizvyvYZkrr3WfVgm4o+YUuQZAFT1LVX9DIC/BrAdwOcBXKmqX1fV+aQKSDkTceWZyth51gfs2ywSc1bolQrwuc01fOeXa7HhTveB9iCD2FmNjxScZwAQkbNE5BsAPg3gowC+DeC/i8hHkioc5VDElWcqCVKzkJXVi8/Lb3uF/tprwH33tS9+02pkD1mPj+Rfqy6gFwH8I4BhVf2Rqt4N4BYAD4hIFjOCUII8eyBiqDxjzMiciTcNlLGixeW383X8VuhBZTk+UkBuAwNmzAArWvzuj71+F+eNg8DZ4GsAkFNEfLHOZbsZN6d5bPjy5S/UtVJpPygb5cfCjzg/4DEILOZ3+TA8PKzT09NpF6PUGg3T7Xzs2Jn7qlVzkZzGFWCjYS6Kh4bydwXaaAArVizuTunpAWZmzP/tx7XoOLdPmm6f7m7MHZ/FRh3H384ubqG4fSaTC0/r6THvOT6+uGGT53NJrYnITlUddt7PbKAUSBQDgFEl6ewkBXNYUSYY3bWruS/95Engy19efFx33eU4Tpjuqbee3o6LlhxoqvyB5s+k3dBBGueSMsCtWZDVG7uA0tfpFPmo5o+nMVU/VNlb9JNs27a4/PbMnX7SAE1NuSdsczsXrdIjZXXZA0UH3BCGotDJAGCU88eTnooYquxtLqvXrDFltuvqaj4uy7loYBg78L6uBvbvB158ETh8uPlxlUrzZ9Jq5g6ndZYXAwAFFnaCTJQVTdJTEZ1lPxcNXLFkB3713F73PiEfEaNWA7ZuNRV2X5/59+tfN/vjON2MSRzAIH6Ma/DSkUG8/39M4p57mh/X1QX8/fYGRlbuaHovr8DNaZ0l5tYsyOqNXUD5FnVXQ5K59exlv3khtcJRVHXeOghnAQKkpLb3EtXrZo9e+3F9YdQ9n8/gbzVv9H5rj0n97NVP5dUj5XYuOcunOJDFTeFhFpj9AsA+AF9o93gGgOwJWklEXWnHWkk5XnxiQvX8SnNl7BrNQkQ7+xhDpWICQb2uqlNTJpe/I6PnVb1Ti17eLfFbkAhrP1zm+imWzAUAAF0A/i9MvqEeAH8PYHWr5zAAZEvYSiLTV5ZW4cbGXA/uN9umdLZvuXsAcF7hh8zg2VR3e/zymbG69vScuetDXVN6orq8dZl8ngIOCheLVwBYmkKvk+WDAPap6isAICJPArgOwJ4Uy0R2LSaG27u4+441cDH2496NQ1i/vuYr5UBa88xbznW3JsovXXpmdNVa8DA6Clx6Kc7GW8CpE+4v7uw4HxkB1q8H9u/Hm/1DePVIDUMN92O3xhjs6yus8ZHa2oUO/IW5/5idBcbHsWGkhpkNZjopAFx2/hB6Lu+8M79lWbg+oFjcokISNwCfBPDXtp9vAfCoy+PuADANYPqCCy6IK0CSU5vLe6uL2+oPfwvL9Siqum9z8L6CpFoELQ/Jz64o1p68PT2q3d1n9k50GwPw+74t3r7pqtvPiYqgj40tgOJBBruAfAUA+41dQAnxUQPU6+794fOVNonkHZLqa257SF67orQKCNu2md26WlTKQSrTUHW3W1CIIKJy87pi8QoAaXYBvQ7gfNvPKxbuow5EspzfRx9ArQb8x037MfvFHgBnHifHj5mNZefnm3MNuJTV6kay97SsX+8ou3VQ/f3AkSOhDq7tIbnNhQTM3ExVYMmS5ieffTawalXL9921yzzVzqs7xdZj5HmIrikhnLkdOuxjazSAlSuBnTtDn27KC7eokMQNwFKYXccuxJlB4H/a6jlsAbQW2dW038vWel3nO9hM1tdMSeugrPdp091yuvyOK2BfhzQ21nwMvb2qP/tZqD4Ra1/egKel7VTN5ctN62u2J/p+Gs7+KSZkrQvIlAkfA/APMLOBNrV7PAOAt8j7bf32AdjTWbabFRO0zK365dv1o7jUYK6HZK9t3XIrWMcQsE/Eq+jtnupVfOfrDWNK33Zs9h5mxk+7MrPvvxgyGQCC3hgAvPm6mg7aN+z38fW66Q93Xu76vEr2rFdb9cu7VXY+xy5OH5KztrWmfno9P8D5cyt6X585Ta1Oo9fbO1+v0zn/fsvcYUyhjGAAKLi2dV8SbfuQI4ee9WrQFkCQGszrhFlBoMNZNNu2Bb+aDpqw7ZbuCdMFF9FILVsAxcUAUAKe9W+Qv+xOZ5BEPafTOig/Uy6DHGe72tZ2DEEOyR5nu7vNjFG/9XO7xWA/3Dyl51fq3l1YEeDsn2JiACgJ1/rA75VxVkcArYNqM+VSVf3XYD6DhWd6Bp8v2dOj+tRTHabKsBVivmrWWsR5VZ7pldoUCgNAmfmp7IrU/vdbg7UJFnv2uOfm94orXkMWvb3BYumi4hfpc6HUeAUApoMuAz9J/IuUFN7vbugt8lpPTpp8/Sdcsj547QXgtZTgxIlg+x4sKn6RPhcKL8qt6GwYAMqiXRL/OJPCx/TljYQzWDQaeOtHO3DvxoZr5W9xq4OtONvb6+/xvjBZP8W4XycDQJm0ujLuZKuvVvK02exCWQc2XIOfHx/ETfAuq1cdPDJiVv86g0DoOjuuzyUPsnzhkJQot9Fz49YvlNUbxwASEOUIYJ76r13KehRVPRdm05WuLjMI7Hd2TOSzaco2MpvVCQlJi2hxBjKYC4iyKMpczXnKK+xS1ll0Y3V1P6a0hk2bgD/4A/+5cfzk9QkkzRzaSfOdJCrn/CTuirkLkF1AFJ889V+7lLUbs/jH2SGcOgU89BBw+eXAvn3+6yC/Y9HkUIaBb79dozF3ATIAUHxy0H99upsZpqxareIQluFdVLER43hjrobZWffuV2cXNbusI5KnC4cwgvbrt5vA0QEGAIpXjF/eTjVdhGEEu797ANf3bccgDuApNJfVuhB1Pveuu/Iz1p15Obhw6EiYFk5MzUkx4wP5MDw8rNPT02kXgwqg0TAVtX14orcX+MlPTCVuv9+uWjV58i+/3Psx1uMOHChOnZWKSDa3yCC3L1/MXxgR2amqw8772QKgyOSpC8TtIuzECeDqq4E//EMTDPr7zWO6uxdfiB450vxcJ68Lujydo9QVdRAlQy0czgKiSEx6bE7Vjq+LvAh2BHNqtWp3fNy81ews8MgjZoMzexkbDffn2rl1WYc9R1RAkU8TC8ltbmhWb1wHkE1hp/v7murdZkewTqbHT0y45/rxcxzOef6f/Wzref95WhJBxQMmg6O4hFmr4qtCbLMfwDNj9Y7XCnklfPNzHM7g0yoYZWKzFXsBg2z2U6YFaAXlFQA4BkAdCzNrz9dECLcHLZhf2o2vfG5/xyvkV60C/uZvTDdsf3/z71sdh7OLulWXddIzG5vGGuzTls47D1ixov2UpTyl8aBw3KJCVm9sAWRX0NQHnbYA5nqretFAPbIrautCN4INwTwltdmKs2vtmbEWLSkrX/WePYtfhH1WhQJ2AVHcgvYW+KoQFx50qrei84DOV8wYwKGxidjqpyD7z4R97bjqUbd6+6reKT01sNw7ALhtWpCJPiuKSrkDAPsxM8vPR/PMWF2v6p3S4b49elXvlLmi1c6uqNu979iYqRMHBvKVi8yt3r6wv65zvS1aAG4RlC2AQilvAGBWwVxrVw+Fie3tvhJjY63rxizzOl+HxmzRsrvb3NqNeHOD4MLwCgDFXgmcwoo7itaOHWYM8tChM/ctW2YyS6xd6/6cVmsLvL4SO3eaJQb9/e47gQ0MmFXCXu+ZJdZ6g+5uM9B8er2B/cS8+Wbzgbr9bRR1NW7JeK0ELvZCsDylIyZXQWfPtFts5faVAExdWKkAx48DIs2ve/JkfnKRea4xsqeUrtXM9CdnpHD+XZQpDXUJsQVAiQl7Mel5Revy+u0+brfH+DE2Btx5p/fvc3uhnNuCUxDlzAWUoZwbZdfJlHK/CUX9rC1wfiV6e83/7ZY62sWjo60r/1xPly9qvh3ypdgtAAuvclKVVEMsyPvY0wt1ktmTjUzKg3K2ACy8yklVuyvzqDJkBmnwWV+JVavatwhapWovw+ZVVFzlCAB+MVdvLFoN5EbdfRJm/xn7c3btav59q0HnjlM88DtHKWIAsOS6IzfbvK7MgWA74wV5v6EhcxXu97W8WgTtho06Gmbid45SVo4xgHbYkZsI51BMmDn+fkSRdz/osFHgYSZ+5yhB5VwH4BfXCyTCOaU8jgyZ9v22rY9zdNTMiw/yUQad/h54ujy/c5QB7AICks/VSwA6n6Xr1n2em0FZfucoAxgAAK4XSFGYQVugufv8gQdMIMhNvcrvHGUAxwDsuF4gF7xW81YqwOOPm//7WTmcCfzOUQI4BuAH857kglc+n+PHTcV/4IC55aJe5XeOUsQAQLnj1s1jWbLEzOW/9lrWq0TtpDIGICKfEpGXRWReRJqaJZRRGVm0ZO8+dzp6FLj+ek6pJ/IjrUHglwBsAPB8Su9PQWVg0ZI9/liDx5s3m/QNdlEtKCMqulQCgKruVdVfpPHeLWXkCjdz7JPro1yyG4Bb/KnVgPvuA773PaCvb/HjMzn1kyhjMj8NVETuEJFpEZluxFnhZOAKN22e8S/lyfXt4s+aNcD8/OLnZHLqJ1HGxBYARGS7iLzkcrsuyOuo6mOqOqyqw7W4RvUycIWbtpbxL+XJ9e3iD6fUE4UT2ywgVV0f12tHruTL8tumT7Bq2HbbB8bET/zx3AaRiDxxGiiQ+hVu2nzFP5caNqk1TH7jD6fUEwWT1jTQG0RkBsA/A/DfRGRbGuU4reR9CL7jn21jnUiHTHwMvodNGUFE3pgKwq7Ey/L9brwORJzJOIrczUTUklcqCAYAOs1v/AuTx9/1tZkTnygR5d4TmHzxu3Vy0CETz+6i3ORuJiomBgAKLMiQScsZtiUffCdKGwMAheJ3ULblRX7JB9+J0sZpoAkq2hizn2mXbS/yOYGfKDVsASSkrJkmfF3k+x18IKJIcRZQArIy2SXNFoj9vQFe8BMlibOAUpSFyS5pt0Csi/zt28OVg4laiaLHAJCAsJNdoqr0spLrLmw50g5eREXFAJCAMJNdoqz0stACCVuOrAQvoiJiAEhIkFw2UVd6WZluH6YcWQleREXEAJAgv5Ndoq70sjLdPkw5kgxeHGegsmEAyKA4Kr2sZNMMWo6kghfHGaiMOA00o4Jk5yyDOKewZmWaLlFcvKaBciVwRnGB7GJxbvZS8g3hqMQYADKMO1wlIyuD5ERJ4xgAlV5WBsmJksYWABHY5UblxABAroqWudQPdrlR2bALiJpwSiRROTAA0CJFSb3ARV1E7TEA0CJFSL1gtWDWrWMLhqgVBgBaJO9TIhsN4LbbTMvl6FHz7223sSVA5IYBgBbJ+5TIXbuaA9jJk+Z+IlqMs4CoCadEEpUDAwC5yuuUyDVrzuRPsnR3m/uJaDF2AVGmBZ3NU6sBW7cClQrQ12f+3bo1n8GMKG4MAJRZYdcjjIwAr70GPPec+bfMWVSJWmE6aMokpmgmio5XOmi2ACiTirAegSjrGAAok/K+HoEoDxgAKJPyvh6BKA84DZQyi+sRiOLFAECZltf1CER5wC4gIqKSYgAgIiopBgAiopJiACAiKikGACKikspVKggRaQA44PKrcwG8mXBxksJjyyceWz4V9dgGVbVpPl2uAoAXEZl2y3NRBDy2fOKx5VORj80Nu4CIiEqKAYCIqKSKEgAeS7sAMeKx5ROPLZ+KfGxNCjEGQEREwRWlBUBERAExABARlVRhAoCIbBaR/y0iu0XkRyLyT9IuU1RE5EER+fnC8f1nETkr7TJFRUQ+JSIvi8i8iOR++p2IfFREfiEi+0TkC2mXJ0oi8riI1EXkpbTLEiUROV9EnhORPQvfxc+lXaakFCYAAHhQVS9R1UsBfB/A/SmXJ0o/BvABVb0EwD8A+LOUyxOllwBsAPB82gXplIh0Afg6gN8HsBrAiIisTrdUkXoCwEfTLkQM5gD8iaquBvAhAP+mYJ+bp8IEAFV9x/ZjH4DCjG6r6o9UdW7hxxcArEizPFFS1b2q+ou0yxGRDwLYp6qvqOpJAE8CuC7lMkVGVZ8H8Ju0yxE1VX1DVV9c+P9hAHsBnJduqZJRqA1hROQvAPxrAIcArEu5OHHZCOCptAtBrs4D8EvbzzMArkipLBSCiAwBWAPgf6VclETkKgCIyHYA73P51SZV/S+qugnAJhH5MwCfBfDniRawA+2ObeExm2Caq99Ksmyd8nNsRGkTkX4AzwC429GjUFi5CgCqut7nQ78F4AfIUQBod2wichuAjwO4WnO2eCPA55Z3rwM43/bzioX7KONEpBum8v+Wqn4n7fIkpTBjACJyse3H6wD8PK2yRE1EPgrgTwF8QlXfTbs85GkHgItF5EIR6QFwM4DvpVwmakNEBMA4gL2q+nDa5UlSYVYCi8gzAH4XwDxMyujPqGohrr5EZB+AXgAHF+56QVU/k2KRIiMiNwD4GoAagLcB7FbVf5lqoTogIh8D8FUAXQAeV9W/SLdE0RGRSQD/AiZl8q8B/LmqjqdaqAiIyFUAfgrg/8DUHwDw71X1B+mVKhmFCQBERBRMYbqAiIgoGAYAIqKSYgAgIiopBgAiopJiACAiKikGAKIAFjJHvioiv73w89kLPw+JyA9F5G0R+X7a5STygwGAKABV/SWAvwLwlwt3/SWAx1R1P4AHAdySUtGIAmMAIAruKwA+JCJ3A7gKwEMAoKo/AXA4xXIRBZKrXEBEWaCqsyLyeQA/BHCtqs6mXSaiMNgCIArn9wG8AeADaReEKCwGAKKARORSANfA7B51j4i8P90SEYXDAEAUwELmyL+CyRn/GszA70PploooHAYAomD+GMBrqvrjhZ//A4BVIvLPReSnAJ4GcLWIzIhIbrOaUjkwGygRUUmxBUBEVFIMAEREJcUAQERUUgwAREQlxQBARFRSDABERCXFAEBEVFL/H+VteQo7i0p6AAAAAElFTkSuQmCC\n", "text/plain": [ - "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", - " ('standardscaler', StandardScaler()),\n", - " ('logisticregression', LogisticRegression())])" + "
" ] }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" } ], "source": [ - "import numpy\n", - "from sklearn.pipeline import make_pipeline\n", - "from sklearn.preprocessing import FunctionTransformer, StandardScaler\n", - "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.datasets import make_classification\n", + "from pandas import DataFrame\n", + "\n", + "X, y = make_classification(200, n_classes=2, n_features=2, n_informative=2,\n", + " n_redundant=0, n_clusters_per_class=2, hypercube=False)\n", "\n", - "pipe = make_pipeline(\n", - " FunctionTransformer(numpy.log),\n", - " StandardScaler(),\n", - " LogisticRegression())\n", - "pipe.fit(X_train, y_train)" + "df = DataFrame(X)\n", + "df.columns = ['X1', 'X2']\n", + "df['y'] = y\n", + "ax = df[df.y == 0].plot.scatter(x=\"X1\", y=\"X2\", color=\"blue\", label=\"y=0\")\n", + "df[df.y == 1].plot.scatter(x=\"X1\", y=\"X2\", color=\"red\", label=\"y=1\", ax=ax);" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Let's convert it into ONNX." + "Split into train and test as usual." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "FunctionTransformer is not supported unless the transform function is None (= identity). You may raise an issue at https://github.com/onnx/sklearn-onnx/issues.\n" - ] - } - ], + "outputs": [], "source": [ - "from mlprodict.onnx_conv import to_onnx\n", - "try:\n", - " onx = to_onnx(pipe, X_train.astype(numpy.float64))\n", - "except RuntimeError as e:\n", - " print(e)" + "from sklearn.model_selection import train_test_split\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Use ONNX instead of numpy\n", - "\n", - "The pipeline cannot be converter because the converter does not know how to convert the function (`numpy.log`) held by `FunctionTransformer` into ONNX. One way to avoid that is to replace it by a function `log` defined with *ONNX* operators and executed with an ONNX runtime." + "The model..." ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", - " ('standardscaler', StandardScaler()),\n", - " ('logisticregression', LogisticRegression())])" + "array([1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1,\n", + " 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1,\n", + " 1, 0, 1, 0, 0, 1], dtype=int64)" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "import mlprodict.npy.numpy_onnx_pyrt as npnxrt\n", + "import numpy\n", + "from sklearn.base import ClassifierMixin, BaseEstimator\n", + "from sklearn.linear_model import LogisticRegression\n", "\n", - "pipe = make_pipeline(\n", - " FunctionTransformer(npnxrt.log),\n", - " StandardScaler(),\n", - " LogisticRegression())\n", - "pipe.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "onx = to_onnx(pipe, X_train.astype(numpy.float64), rewrite_ops=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%onnxview onx" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The operator `Log` is belongs to the graph. There is some overhead by using this function on small matrices. The gap is much less on big matrices." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4.43 \u00b5s \u00b1 311 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" - ] - } - ], - "source": [ - "%timeit numpy.log(X_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "16 \u00b5s \u00b1 2.13 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" - ] - } - ], - "source": [ - "%timeit npnxrt.log(X_train)" + "class TwoLogisticRegression(ClassifierMixin, BaseEstimator):\n", + " \n", + " def __init__(self):\n", + " ClassifierMixin.__init__(self)\n", + " BaseEstimator.__init__(self)\n", + " \n", + " def fit(self, X, y, sample_weights=None):\n", + " if sample_weights is not None:\n", + " raise NotImplementedError(\"weighted sample not implemented in this example.\")\n", + " \n", + " # Barycenters\n", + " self.weights_ = numpy.array([(y==0).sum(), (y==1).sum()])\n", + " p1 = X[y==0].sum(axis=0) / self.weights_[0]\n", + " p2 = X[y==1].sum(axis=0) / self.weights_[1]\n", + " self.centers_ = numpy.vstack([p1, p2])\n", + " \n", + " # A vector orthogonal\n", + " v = p2 - p1\n", + " v /= numpy.linalg.norm(v)\n", + " x = numpy.random.randn(X.shape[1])\n", + " x -= x.dot(v) * v\n", + " x /= numpy.linalg.norm(x)\n", + " self.hyperplan_ = x.reshape((-1, 1))\n", + " \n", + " # sign\n", + " sign = ((X - p1) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel()\n", + " \n", + " # Trains models\n", + " self.lr0_ = LogisticRegression().fit(X[sign == 0], y[sign == 0])\n", + " self.lr1_ = LogisticRegression().fit(X[sign == 1], y[sign == 1])\n", + "\n", + " return self\n", + " \n", + " def predict_proba(self, X):\n", + " sign = self.predict_side(X).reshape((-1, 1))\n", + " prob0 = self.lr0_.predict_proba(X)\n", + " prob1 = self.lr1_.predict_proba(X)\n", + " prob = prob1 * sign - prob0 * (sign - 1)\n", + " return prob\n", + " \n", + " def predict(self, X):\n", + " prob = self.predict_proba(X)\n", + " return prob.argmax(axis=1)\n", + "\n", + " def predict_side(self, X):\n", + " return ((X - self.centers_[0]) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel()\n", + " \n", + " \n", + "model = TwoLogisticRegression()\n", + "model.fit(X_train, y_train)\n", + "model.predict(X_test)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Slightly more complex functions with a FunctionTransformer\n", - "\n", - "What about more complex functions? It is a bit more complicated too. The previous syntax does not work." + "Let's compare the model a single logistic regression. It shouuld be better. The same logistic regression applied on both sides is equivalent a single logistic regression and both half logistic regression is better on its side." ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", - " ('standardscaler', StandardScaler()),\n", - " ('logisticregression', LogisticRegression())])" + "(0.82, 0.84)" ] }, - "execution_count": 12, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def custom_fct(x):\n", - " return npnxrt.log(x + 1)\n", - "\n", - "pipe = make_pipeline(\n", - " FunctionTransformer(custom_fct),\n", - " StandardScaler(),\n", - " LogisticRegression())\n", - "pipe.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "FunctionTransformer is not supported unless the transform function is of type wrapped with onnxnumpy.\n" - ] - } - ], - "source": [ - "try:\n", - " onx = to_onnx(pipe, X_train.astype(numpy.float64), rewrite_ops=True)\n", - "except TypeError as e:\n", - " print(e)" + "from sklearn.metrics import accuracy_score\n", + "lr = LogisticRegression().fit(X_train, y_train)\n", + "accuracy_score(y_test, lr.predict(X_test)), accuracy_score(y_test, model.predict(X_test))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The syntax is different." + "However, this is true in average but not necessarily true for one particular datasets. But that's not the point of this notebook." ] }, { "cell_type": "code", - "execution_count": 13, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", - " ('standardscaler', StandardScaler()),\n", - " ('logisticregression', LogisticRegression())])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from typing import Any\n", - "from mlprodict.npy import onnxnumpy_default, NDArray\n", - "import mlprodict.npy.numpy_onnx_impl as npnx\n", - "\n", - "@onnxnumpy_default\n", - "def custom_fct(x: NDArray[(None, None), numpy.float64]) -> NDArray[(None, None), numpy.float64]:\n", - " return npnx.log(x + numpy.float64(1))\n", - "\n", - "pipe = make_pipeline(\n", - " FunctionTransformer(custom_fct),\n", - " StandardScaler(),\n", - " LogisticRegression())\n", - "pipe.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
\n", - "" - ], "text/plain": [ - "" + "array([[-0.01589338, -0.11623031],\n", + " [-0.37916406, 0.41219093]])" ] }, - "execution_count": 15, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "onx = to_onnx(pipe, X_train.astype(numpy.float64), rewrite_ops=True)\n", - "%onnxview onx" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's compare the time to *numpy*." + "model.centers_" ] }, { "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.34 \u00b5s \u00b1 522 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" - ] - } - ], - "source": [ - "def custom_numpy_fct(x):\n", - " return numpy.log(x + numpy.float64(1))\n", - "\n", - "%timeit custom_numpy_fct(X_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "17.8 \u00b5s \u00b1 722 ns per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit custom_fct(X_train)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The new function is slower but the gap is much less on bigger matrices. The default ONNX runtime has a significant cost compare to the cost of a couple of operations on small matrices." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "349 \u00b5s \u00b1 21.4 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" - ] - } - ], - "source": [ - "bigx = numpy.random.rand(10000, X_train.shape[1])\n", - "%timeit custom_numpy_fct(bigx)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "367 \u00b5s \u00b1 50.1 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" - ] - } - ], - "source": [ - "%timeit custom_fct(bigx)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Function transformer with FFT\n", - "\n", - "The following function is equivalent to the module of the output of a FFT transform. The matrix $M_{kn}$ is defined by $M_{kn}=(\\exp(-2i\\pi kn/N))_{kn}$. Complex features are then obtained by computing $MX$. Taking the module leads to real features: $\\sqrt{Re(MX)^2 + Im(MX)^2}$. That's what the following function does." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### numpy implementation" - ] - }, - { - "cell_type": "code", - "execution_count": 19, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178631, 2.491644 , 1.3178631],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.1368184]], dtype=float32)" + "array([[-0.82405569],\n", + " [-0.5665088 ]])" ] }, - "execution_count": 20, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def custom_fft_abs_py(x):\n", - " \"onnx fft + abs python\"\n", - " # see https://jakevdp.github.io/blog/\n", - " # 2013/08/28/understanding-the-fft/\n", - " dim = x.shape[1]\n", - " n = numpy.arange(dim)\n", - " k = n.reshape((-1, 1)).astype(numpy.float64)\n", - " kn = k * n * (-numpy.pi * 2 / dim)\n", - " kn_cos = numpy.cos(kn)\n", - " kn_sin = numpy.sin(kn)\n", - " ekn = numpy.empty((2,) + kn.shape, dtype=x.dtype)\n", - " ekn[0, :, :] = kn_cos\n", - " ekn[1, :, :] = kn_sin\n", - " res = numpy.dot(ekn, x.T)\n", - " tr = res ** 2\n", - " mod = tr[0, :, :] + tr[1, :, :]\n", - " return numpy.sqrt(mod).T\n", - "\n", - "x = numpy.random.randn(3, 4).astype(numpy.float32)\n", - "custom_fft_abs_py(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### ONNX implementation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This function cannot be exported into ONNX unless it is written with ONNX operators. This is where the numpy API for ONNX helps speeding up the process." + "model.hyperplan_" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + "(array([[-0.63926519, 0.37923185]]), array([[-0.90877675, 1.7896483 ]]))" ] }, - "execution_count": 21, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from mlprodict.npy import onnxnumpy_default, onnxnumpy_np, NDArray\n", - "import mlprodict.npy.numpy_onnx_impl as nxnp\n", - "\n", - "\n", - "def _custom_fft_abs(x):\n", - " dim = x.shape[1]\n", - " n = nxnp.arange(0, dim).astype(numpy.float32)\n", - " k = n.reshape((-1, 1))\n", - " kn = (k * (n * numpy.float32(-numpy.pi * 2))) / dim.astype(numpy.float32)\n", - " kn3 = nxnp.expand_dims(kn, 0)\n", - " kn_cos = nxnp.cos(kn3)\n", - " kn_sin = nxnp.sin(kn3)\n", - " ekn = nxnp.vstack(kn_cos, kn_sin)\n", - " res = nxnp.dot(ekn, x.T)\n", - " tr = res ** 2\n", - " mod = tr[0, :, :] + tr[1, :, :]\n", - " return nxnp.sqrt(mod).T\n", - "\n", - "\n", - "@onnxnumpy_default\n", - "def custom_fft_abs(x: NDArray[Any, numpy.float32],\n", - " ) -> NDArray[Any, numpy.float32]:\n", - " \"onnx fft + abs\"\n", - " return _custom_fft_abs(x)\n", - "\n", - "\n", - "custom_fft_abs(x)" + "model.lr0_.coef_, model.lr1_.coef_" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "`custom_fft_abs` is not a function a class holding an ONNX graph. A method `__call__` executes the ONNX graph with a python runtime." + "Let's draw the model predictions. Colored zones indicates the predicted class, green line indicates the hyperplan splitting the features into two. A different logistic regression is applied on each side." ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 10, "metadata": { "scrolled": false }, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%onnxview custom_fft_abs.compiled.onnx_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Every intermediate output can be logged." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "-- OnnxInference: run 39 nodes\n", - "Onnx-Shape(x) -> Sh_shape0\n", - "+kr='Sh_shape0': (2,) (dtype=int64 min=3 max=4)\n", - "Onnx-Slice(Sh_shape0, Sl_Slicecst, Sl_Slicecst1, Sl_Slicecst2) -> Sl_output01\n", - "+kr='Sl_output01': (1,) (dtype=int64 min=4 max=4)\n", - "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst\n", - "+kr='Sq_Squeezecst': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Squeeze(Sl_output01, Sq_Squeezecst) -> Sq_squeezed01\n", - "+kr='Sq_squeezed01': () (dtype=int64 min=4 max=4)\n", - "Onnx-Identity(Sl_Slicecst2) -> Su_Subcst\n", - "+kr='Su_Subcst': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Sub(Sq_squeezed01, Su_Subcst) -> Su_C0\n", - "+kr='Su_C0': (1,) (dtype=int64 min=4 max=4)\n", - "Onnx-ConstantOfShape(Su_C0) -> Co_output01\n", - "+kr='Co_output01': (4,) (dtype=int64 min=1 max=1)\n", - "Onnx-Identity(Sl_Slicecst2) -> Cu_CumSumcst\n", - "+kr='Cu_CumSumcst': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-CumSum(Co_output01, Cu_CumSumcst) -> Cu_y0\n", - "+kr='Cu_y0': (4,) (dtype=int64 min=1 max=4)\n", - "Onnx-Add(Cu_y0, Ad_Addcst) -> Ad_C01\n", - "+kr='Ad_C01': (4,) (dtype=int64 min=0 max=3)\n", - "Onnx-Cast(Ad_C01) -> Ca_output0\n", - "+kr='Ca_output0': (4,) (dtype=float32 min=0.0 max=3.0)\n", - "Onnx-Reshape(Ca_output0, Re_Reshapecst) -> Re_reshaped0\n", - "+kr='Re_reshaped0': (4, 1) (dtype=float32 min=0.0 max=3.0)\n", - "Onnx-Mul(Ca_output0, Mu_Mulcst) -> Mu_C01\n", - "+kr='Mu_C01': (4,) (dtype=float32 min=-18.84955596923828 max=-0.0)\n", - "Onnx-Mul(Re_reshaped0, Mu_C01) -> Mu_C0\n", - "+kr='Mu_C0': (4, 4) (dtype=float32 min=-56.548667907714844 max=-0.0)\n", - "Onnx-Cast(Sq_squeezed01) -> Ca_output01\n", - "+kr='Ca_output01': () (dtype=float32 min=4.0 max=4.0)\n", - "Onnx-Div(Mu_C0, Ca_output01) -> Di_C0\n", - "+kr='Di_C0': (4, 4) (dtype=float32 min=-14.137166976928711 max=-0.0)\n", - "Onnx-Identity(Sl_Slicecst2) -> Un_Unsqueezecst\n", - "+kr='Un_Unsqueezecst': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Unsqueeze(Di_C0, Un_Unsqueezecst) -> Un_expanded0\n", - "+kr='Un_expanded0': (1, 4, 4) (dtype=float32 min=-14.137166976928711 max=-0.0)\n", - "Onnx-Cos(Un_expanded0) -> Co_output0\n", - "+kr='Co_output0': (1, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", - "Onnx-Sin(Un_expanded0) -> Si_output0\n", - "+kr='Si_output0': (1, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", - "Onnx-Concat(Co_output0, Si_output0) -> Co_concat_result0\n", - "+kr='Co_concat_result0': (2, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", - "Onnx-Transpose(x) -> Tr_transposed0\n", - "+kr='Tr_transposed0': (4, 3) (dtype=float32 min=-1.8227919340133667 max=2.354652166366577)\n", - "Onnx-MatMul(Co_concat_result0, Tr_transposed0) -> Ma_Y0\n", - "+kr='Ma_Y0': (2, 4, 3) (dtype=float32 min=-2.7854056358337402 max=3.9340782165527344)\n", - "Onnx-Pow(Ma_Y0, Po_Powcst) -> Po_Z0\n", - "+kr='Po_Z0': (2, 4, 3) (dtype=float32 min=0.0 max=15.476971626281738)\n", - "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst3\n", - "+kr='Sl_Slicecst3': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst4\n", - "+kr='Sl_Slicecst4': (1,) (dtype=int64 min=1 max=1)\n", - "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst5\n", - "+kr='Sl_Slicecst5': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Slice(Po_Z0, Sl_Slicecst3, Sl_Slicecst4, Sl_Slicecst5) -> Sl_output0\n", - "+kr='Sl_output0': (1, 4, 3) (dtype=float32 min=0.7225952744483948 max=15.476971626281738)\n", - "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst1\n", - "+kr='Sq_Squeezecst1': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Squeeze(Sl_output0, Sq_Squeezecst1) -> Sq_squeezed0\n", - "+kr='Sq_squeezed0': (4, 3) (dtype=float32 min=0.7225952744483948 max=15.476971626281738)\n", - "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst6\n", - "+kr='Sl_Slicecst6': (1,) (dtype=int64 min=1 max=1)\n", - "Onnx-Identity(Sl_Slicecst1) -> Sl_Slicecst7\n", - "+kr='Sl_Slicecst7': (1,) (dtype=int64 min=2 max=2)\n", - "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst8\n", - "+kr='Sl_Slicecst8': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Slice(Po_Z0, Sl_Slicecst6, Sl_Slicecst7, Sl_Slicecst8) -> Sl_output02\n", - "+kr='Sl_output02': (1, 4, 3) (dtype=float32 min=0.0 max=1.636295199394226)\n", - "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst2\n", - "+kr='Sq_Squeezecst2': (1,) (dtype=int64 min=0 max=0)\n", - "Onnx-Squeeze(Sl_output02, Sq_Squeezecst2) -> Sq_squeezed02\n", - "+kr='Sq_squeezed02': (4, 3) (dtype=float32 min=0.0 max=1.636295199394226)\n", - "Onnx-Add(Sq_squeezed0, Sq_squeezed02) -> Ad_C0\n", - "+kr='Ad_C0': (4, 3) (dtype=float32 min=0.8215643763542175 max=17.113265991210938)\n", - "Onnx-Sqrt(Ad_C0) -> Sq_Y0\n", - "+kr='Sq_Y0': (4, 3) (dtype=float32 min=0.9064018726348877 max=4.1368184089660645)\n", - "Onnx-Transpose(Sq_Y0) -> y\n", - "+kr='y': (3, 4) (dtype=float32 min=0.9064018726348877 max=4.1368184089660645)\n" + ":20: MatplotlibDeprecationWarning: shading='flat' when X and Y have the same dimensions as C is deprecated since 3.3. Either specify the corners of the quadrilaterals with X and Y, or pass shading='auto', 'nearest' or 'gouraud', or set rcParams['pcolor.shading']. This will become an error two minor releases later.\n", + " return ax.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)\n" ] }, { "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABF00lEQVR4nO29eZgcdbX//zq9Tc8MJJlMJoSsEzYTiZAQyBBkiSwS81VQBA244DUY4Iq4kavITzEgyFVEhSiQS1BABa9BIF4JGJBVIJBAhIRE2YZshEwmk2223j6/P7p70t1T3dN71XSf1/PwMFNdXfWpTk+dOud9FjHGoCiKoijpcNm9AEVRFMXZqKFQFEVRMqKGQlEURcmIGgpFURQlI2ooFEVRlIx47F5AKRgxYoRpbm62exmKoiiDhtWrV+8wxjRZvVaRhqK5uZlVq1bZvQxFUZRBg4i8m+41DT0piqIoGVFDoSiKomREDYWiKIqSkYrUKBRFqQyCwSCbN2+mp6fH7qVUDH6/n7Fjx+L1erN+jxoKRVEcy+bNmznwwANpbm5GROxezqDHGEN7ezubN29m4sSJWb/PttCTiIwTkSdE5HURWSciX7fYZ5aI7BaRNbH/fmDHWhVFsYeenh4aGxvVSBQJEaGxsTFnD81OjyIEfNsY87KIHAisFpEVxpjXU/Z7xhjzcRvWpyjOp60NWluhuRmaLFPgBz1qJIpLPp+nbR6FMeY9Y8zLsZ/3AuuBMXatR1EGHffeCxMmwBlnRP9/7712r0ipUByR9SQizcA0YKXFyzNF5J8islxEjsxwjPkiskpEVrW1tZVqqYriDNraYN486O6G3buj/583L7pdKSqtra1MmTLF7mUAcMABB9hyXtvFbBE5ALgf+IYxZk/Kyy8DE4wx+0RkDvAgcLjVcYwxi4HFAIc11pplF0wu3aIVxWaGtXczM9RLYt5KMNTL858/jl2Ntbatq9iMuXQRu94Ol/Qc4VFHZHy9oztAOGJo7wqUdB0AoVAIjyfzbTnTOhrrfMVeEmCzRyEiXqJG4vfGmD+nvm6M2WOM2Rf7+WHAKyIjyrxMx7PB383S4e1s8HfbvRSlTHTVe5FI8nRKiRi66rNPeaxUXmxbw02v3cGLbWuKdsxwJMw3vnopHz52Kud+Yg4bXn+dj5zQ0vf6W2++0ff7tMlH8MOrruSk447hjJM/zNtvvQnAjrY2vnTBZzn9pBM4/aQTWPn8cwD893XXcum8/2DOabO4dN5/cO89d/P5z3yas2afwXFHfZCfXP+jfuvZt28fn5pzJh85oYWTjjuGh/9vGRD1fiZPnsxXvvIVjjzySD760Y/S3V34fcHOrCcBlgDrjTE3pdlnVGw/RGQG0fW2l2+VzmeDv5vvj9vI70e08f1xG9VYVAkBv4c1LaMIuYWg10XILaxpGUXAb3uQwFZebFvDJ1d8hevX3MInV3xlQGMxkDcR5+0332Te/Ev4x6o1DB02jFf/uYYhQ4fy2j//CcC999zNBV/4Yt/+Q4YO5ZmXXuaiiy/lqv+6AoDvLfg2l1x2OY898xy//cN9fOOrl/Tt/68N67n//5bzP3fdA8DLq1bx29/fx9MrV7Psz/fzysurk9bj9/u5+74/8cRzK3lw+d+4+srvEB9r/cYbb/DVr36VdevWMWzYMO6///6srjETdn6rPgx8AXhNRNbEtn0PGA9gjLkNOBe4VERCQDcw1+iQ7yTW1nUREkNEIIRhbV0Xk3oqJ/SgpGfrhKHsOKieus4gXfXeqjcSAM9uW0UgHCBMhEAkyLPbVjGjaWrBxx3f3MyHjj4agKOnHsOmje/y+Qv/g3vvuYsPTvkpD97/J/721D/69j/nvM9E//+Zz/L/fXcBAE8/8Xf+vWF93z579+xl3759AMz+fx+ntnb/3+2sU09jeGMjAP/v7E+y8rl/MO2Y6X2vG2P40Q+/z/PPPovL5eK9rVvZ/v771Llg4sSJTJ0avebp06fT2tpa8PXb9s0yxjwLZMzTMsYsAhaVZ0WDkylddXiMEMLgMcKUrjq7l6SUkYDfowYigRNHHYvP7SMQCeJzeTlx1LFp983WmwCo8dX0/exyuwj1hPjEJz/FT3/8I06cNYujpx3Td2OH5BTU+M8RE+GRJ5/B7/f3O35dXfLfbWoKa+rvS++7lx07dvD4P17A6/UybfIR9Pb2UFfro6Zm/1rdbvfgDj0pxWFSTy3XbhrP53Y0ce2m8UzqqVXNQqlaZjRN5cEz/ofvHX0ZD57xP2m9iVyMRDr8fj+nnn4GC75+Oed//otJrz24dCkADyz9E8fOiGoXs049nf+59Vd9+8TDVlY8+ffH6di5k+7ubpb/ZRkzZp6Q9PqePbtpamrC6/XyzFNPsmlj2g7hRUEfRSqAST21feGmuGYRkqiHETceilItzGiaWpRwUzac+9nz+euyZXzk9DOStu/a1cHJM6bjq6lh8W/vBuDHN97Ef33r65w8YzqhcIiZHz6Rn938K6vDcsyxx/Klz81l65bNnDf3gqSwU/y8nzvvHE467himHjOdwz/wgdJcYAypxJD/YY215qYzm+1ehi0sHd7O70e0ERFwGfjcjibO3dk48BsVxYGMuXQRh445qKjHLIY3EWfRL25i7549XPmDH/Ztmzb5CB575jkaR+SXoHnvPXez5pXV/PdNv8z5vdmmx65fv57Jk5NLCERktTHGMlanHkWFoZqFopSHL849j9a33+aBhx+1eyklRz2KCmSDv5u1dV1M6arTsJMyqCm2R1FMb8KJqEehZE2qZqFGQ6l2Kt1AlBo1FBWMCtuKohQDTY+tYJKK8SRajKco1YZ6E4WjhqKCiQvbLoMK24qi5I2GniqYeDFeokahmoVSTdjlTbzb+g5fufALdOxs56hpx3DrHb/B5ytNZ9dyoB5FhTOpp5Zzdzb2GQltIKhUC3aGnK75/lVcctnlvPTaeoYNG8bv7vqNbWspBmooqgjVLJRqYEe7m5dfrWVHEWY4/fjahdy26Oa+36/74Q+4/Ve3ZHyPMYZnnnqSsz51DgBzP/cFlv9lWeGLsRENPVURWoynVDpLlw3h8ivH4vEJoSD88tYw55wXyft4n/vihVx4/me55LLLiUQiPLD0f3ng4UeZdfxxlvvf/pu7GdHUxNChQ/sGEI0eM4b3tm7New1OQA1FFWGlWShKpbCj3c3lV46lu8cFPdFtX7/UzcmzIoxoyu+Y4yc0M3z4cF5ds4a27e/zoaOnMm78BJ584aW072nfsSO/kzkYNRRVhhbjKZXKxi0+vF5Dd8/+bR4vbNwojGjKvwPF57/0Ze773d28v/19LvjChezdu5dPnHGq5b63/+Zujpg0id27d/eNNd26ZQsHjx6d9/mdgBqKKkWL8ZRKY/yYAMFQsuwaCsL48YW1Kfp/Z53NDT9aSCgYZPFv7sbtdmf0KABOPPkUlj3wZ8457zPc9/t7+NjHP1HQGuzGzlGo40TkCRF5XUTWicjXLfYREblZRN4UkVdF5Bg71lqJqLCtVBLhUUfQcOSh/PLWMLW1hgOHGGprDb+8NZx32CmOz+fjxJNP4exzzsXtdmf1nh9cex233vJLjvvQZDp27uRzF/5HYYuwGTs9ihDwbWPMyyJyILBaRFYYY15P2OdjwOGx/1qAW2P/VwpEhW2lEjnnvAgnz4qwcaMwfrwp2EgARCIRVr/4Ikt+94es39M88RBWPP2PgXccJNg5CvU94L3Yz3tFZD0wBkg0FGcDd8fmZL8gIsNE5ODYe5UC0GI8pVJIrZcY0URBmkQi/1q/ngvO/SRzPnE2hx52eFGOORhxhEYhIs3ANGBlyktjgE0Jv2+ObetnKERkPjAfoKnOEZfleHQynmInvp4QdZ1Buuq9jp37/YHJk1m97l92L8N2bP/XEZEDgPuBbxhj9uR7HGPMYmAxROdRFGl5VUOSZkFUs1BDoZSK0a27mfriNoxLkIhhTcsotk4YmvNxtOFfebC1MltEvESNxO+NMX+22GULMC7h97GxbUqR0QaCSrnw9YSY+uI2PGGDNxjBEzZMXbkNX08op+OokSgftnkUIiLAEmC9MeamNLstAy4TkfuIiti7VZ8oDVqMp5SLus4gxiUQ3u/4G5dQ1xl0bAiq2rHzX+XDwBeA10RkTWzb94DxAMaY24CHgTnAm0AXMLhzzBxOomYBlV2QNxji45VKV70XiSRHhyVi6Kr3Zn0M9SbKi51ZT88CMsA+BvhqeVakJFLJ4nax4uNKfgT8Hta0jGLqyuR/g0oy2Hfc9mtu/9UtvPP22/zr3S00jhhh95IKonL+ZZSiUqnidmJ8PB76mLpyGzsOqq+oG5XT2TphKDsOqs/LqxsM3sSM40/gox+bw9mzP2r3UoqC/mUollRqQZ7Gx51DwO8piYGQtjZcG98lMn4CpqmwirsfX7uQhoYGLrnsciDaZnxEUxMXf/VrGd931NSpBZ3XaehfhmJJqrgNsHR4+6DXK4oRH1eci+9//8gB/3kxxutFgkH23Xo7gfM+m/fx8mkz/oHJk/M+n1NRQ6GkJS5uV5JeUQ3x8UokG29C2to44D8vRrq7ke7o9MYDLr2Yjlmn5u1Z5NNmvBLRvw5lQCpNrygkPq44F9fGd6OeRPf+Eb/G68W18V3CBYSgcm0zrh6FUpVUol6Ra3xcsY9stYnI+AlIMJi0TYJBIuMnFHT+fNqMVxo6M1sZkLhe8bkdTYM67KQMPsyQkdnv29TEvltvx9TWEhkyBFNbG/29QEE7nzbji3+9iA8dfghbt2zm5JZj+fp/XlLQGuxGH6mUrKimYjxl8BI477N0zDq1aFlPkF+b8fn/eRnz//Oygs/tFNRQKDlTSeK24lw6F6wAOnJ+n2lqKkiTSETbjEdRQ6HkTKWJ2/mgLUCqA20zHkW/4UrOVKK4nQt2twCpBiMV9SaiGGOI9hBVikG0M1JuVOa3TCkp1Twdz+4WIHYbqXKQaCR6jJu9uzo4cFiDGosiYIyhvb0dv9+f0/vUUCh5Ua3T8exsAWK3kbKDrZEDoG0n/h1tdi9lULDdN/D3wO/3M3bs2JyOW5nfLqWsVJNmYWcLkErvU5XoScQJi4tNZgjozMqsOH9ybgYgW7SOQimYapqOF28BEnILQa+LkFvK1gJE+1QpdmHrY4iI3Al8HNhujJli8fos4CHgndimPxtjrinbApWsqLbpeHa1AKnkPlVW3oTiHOz+hv0WWATcnWGfZ4wxHy/PcpR8SdUsKt1o2NUCRPtUKXZg67fMGPO0iDTbuYZqo9SpldUkbNtFpfWpUm/C+QyGb9tMEfknsBW4whizzmonEZkPzAdoqhsMl1V+ypFaWU3CtlI4aiQGB04Xs18GJhhjjgZuAR5Mt6MxZrEx5lhjzLFDKuhpq1gkplZ6gxE8YcPUldvw9YSKep5qErYV+6npaGf4ujXUdLTbvZSKxtF3VGPMnoSfHxaRX4vICGPMDjvXNRixSq0EGNLRw46DDyjaeaqhGK8aKqPLQaHexPjlD9JyzQIiXi+uYJCVV9/IxtlnF2l1SiKO/paLyCjgfWOMEZEZRD0gfXTIA6vUSnfYMOPpzaw5/uCihqAquRjPKnyn4nL5qelop+WaBXh6e6C3B4CWhVfwfsuJ9DY02ry6ysPu9Nh7gVnACBHZDFwNeAGMMbcB5wKXikgI6AbmmnwalVQxiU+/a1pGMW3lNlxhgwACeCKlre6tJM3CqjJ62vPvgUuIVHBLjVJQqDdRv3UTEa+3z0gARDxe6rduUkNRAuzOejp/gNcXEU2fVfLA6un3xZPGcNwzW6I3uxilrO6tpAaCVuE7lwEJG9xV0lKjGBRDwO4cPQ5XyjQ7VyhI5+hxBR9b6Y/TxWwlT9KJ1111/at4S1ndmzodD2Dp8HY2+LsHeKfzsArfpRI3ukpp6W1oZOXVNxKq8ROoP5BQjZ+VV9+o3kSJ0MeeCiVdXyBvKFL26t64ZjHY9YqA38PaY5o4+qXtxPuYpvYzLZbRrUTBvNipsBtnn837LSdSv3UTnaPHJRmJPR0u2ra6aRodZkhDpKjnrUYq4xuo9CNTX6BdjbW2CLCVoFd01fv6bTNA2A1QnL5Ppax32d3TwPbOMYys38JQf+7T45xGb0NjPy/iueW1LL6mAbcXwkGYf3UHJ8wefB6sk1BDUaEM1BfIjureStAr0k1EeP3okWydMKTgz7SUrcSfap3Dohd/hMcVIhTx8LWWqzh5wvKCjpkt5Sqs29PhYvE1DQR6XdAb3bZ4YQNTWnrVsygANRQVjNP6AlVC88DdDX4iAu4EZy0iFMVIQOlaie/uaWDRiz8iEK4lEI5uu2XldRx90Asl9yzKWX3dttWN20ufkQBwe6Lb1VDkjxqKCsdpfYESayxg8DUQDPg9vDLzYKa+8B6IgDGsOf7gon3GpWolvr1zDB5XqM9IjKCNSaxjb8cIhh48+ENQcZpGhwmn5BKEQ9HtSv445w6iVB2DVdwupadWqlbiI+u3EIpEjzGXe1nCPIJhL/XP7GNNy0Elq/0ody+nIQ0R5l/dweKFDbg9USMx/+oO9SYKRA2FYhv5iNtOyQYqpadWCkM01N/B11qu4t4XvsGSyDzq6Aa6IVx5tR8nzO5mSkuvZj0Vkcr4Zig54ZSbbbbidny9Q3b2MOWV7SXtfusUSmGITp6wnFN9j+F+JggJkZhSFVza2Rl2SENEDUQRUUNRZZSj1Xi2ZNNAsG+9Ap5QtPVIsbOBqglXQzdukuP1pSi41PbhlYX+hVURxU69LIZnkqmB4A1vjWZOfL0WlLL1SKVS6nGqaiAqE/0LqyKKmXpZCs8kVbNY7+u0bI0ep5StRyoZp6VNK85Hez1VEcVKvSzVEKTUoUeTA/X91muAoEcIuYtTBV2tBPwedjXWFvXzU2+ictG/siqiWGGHUhWFpWoWh0Zq+6137TFN7GmozetJ2CkifiWiRqKy0b+WKqMYYYdSFYVBf81i6bQQ0yaOZnqbp6AbvJNEfEUZbGjoqQopNOwQ90xCbiHodZUkDBQXtn8/oo3vHrqVF8aQ9/HLNS+8WlFvovKxe8LdncDHge3GmCkWrwvwS2AO0AV8yRjzcnlXqVhRakG0mJ1mSxUqU5RqwW6P4rfA7Ayvfww4PPbffODWMqxJyZJSCKJxUoXtQjrNljJUVu2oN1Ed2D0K9WkRac6wy9nA3bE52S+IyDAROdgY8155VqjYRVzYXu/by3FtPg4xXl4dll8DwUJEfBXA01MKI6EDh5yJ07/5Y4BNCb9vjm3rZyhEZD5Rr4OmOqdflpINp24I8K0XOzAuYeXB27j6ixBwk1cDwXxCZSqAlxcdOORc7A49FQ1jzGJjzLHGmGOH6JPfoCdVgH5mPARd0dkP4ZhmkSu5hMqcKID7ekIMa+92hAhfbG8iceBQ9z4XgV4Xt/9wOFve0b9lJ+D0f4UtwLiE38fGtikVTqoAPasVfGEIGPBFYNqu/iNJS3l+sFcAT/JuwhH+fWQjGw9rqIhw2J4OF688W4PLQ9LAoWAArpw7kksWqmdhN07/li0DLhOR+4AWYLfqE9VBqgA9czM8fhc82QwnbhbMER52JYxKLraW4CQB3KpH1+TX2vnA6zt5pczhsGJ7Es8tr+X2hQ2ICwI9qYNmhVBQdJSpA7A7PfZeYBYwQkQ2A1cDXgBjzG3Aw0RTY98kmh77H/asVCk3cQF62sptuMLRrrEzN0f/C7nhlg+HeGVYO1O66jh1Q6DoWkKpm+flgpV3I4A7Fg4rdQfd3T0NbO8cw8j6LUW9YezpcPHrHzQQCSdGwOPXuN9oGKOjTO3G7qyn8wd43QBfLdNyFIcRF6DHv9nBEevaMW4XEjHc/dHhfPfQrYTE4DXCzMcNnjBFbz/ulOZ5Vt5NnFKHw55qncOiF3+ExxUi5Kpn/nHFCwM9fE89kXCqF9GfYEDw11lfv1IenB56UqqcgN/Dm1Oa2HhYQ98N+8nRu/uK8YLG8NRE4aSNyVrC0I4egj53wTd4J8wc7/NuXngPdyTxWbu04bDdPQ0sevFHBMK1fbO2ixUG2tPh4q+/O5Dkq4ni9hjCof3bfTWGnq6BDYpSOtRQKIOCxBt24mQ8L8Kst5OfNt3BCDOe2UKkwFCUk2oo4t7NhDc7OOL1nUnXVqq1be8cg8cV6jMSAG5PccJAbVvdeGuiM60TcbnB5SJltBI0jU7dopQTNRTKoCOxy+y0XT5mbtnC82OjQvesVjh+M7jCBncBoah0NRR2Go+A38MbU5p4N8G7KuUaRtZvIRRJPn44VJybdtPoMJF+Wb6GL323g7p6w+KFDbg90fPNv7pD9QmbUUOhDEriXWaHtXfz3HjhzM8ZAu5oCu1jd8EJm/fvm2scP90kQE8gkvfM7mIamHKFw4b6O/hay1XcsvrGot+0hzREmH91B4sXNuByQygIFy7YxWmfjtbHTGnp1QptB6GGQhnUdNV7WT0+aiTCrmidxZPNyYYi1zi+ZQ2FwIde3o47kvsY2cFc4T190be4uWOb5U07U7uNbFpxnDC7O61BGNIQUQPhINRQVDhOirOXiokyBF94T18x3lF7fBgCfa+/e8iQnK49XQ1FxCW4E+5d2Xgq6byTt4eNYVOomZH1Wxjq78h6beUkXjNhddPO1G4jl1YcahAGB5V55xjEJN7YgYJu8qV4knWS4Um8vvHvwP0z6jjENZTZ67Yl5dJMeHsPb0xpynq9VjUUa49pYsrLbUn7ScQQ9LgY1t6d9vOw8k5CxssvHrmJf7qnEop4+FrLVZw8YXlen4EdJLbbiFdSx7OhgLSvqUEYvKihcBCJNz5XOAIGIh5XXjf5dE+yhdQXlDKEkqsBSr2+EzfC8Vt6eOmk4RiX8PzBpk/cPvb93GsNrGooQl53kvF4d+IQTnm0NePnYeWd1EQCtLCS5yMnAXDLyus4+qAXHOVZpKvArulox/PsexzkcbOp96C+7fFsKAC3l6RWHMXKlFLsQw2FQ7C6sUM01RNyv8kXu1dRKQxPnHwMkNX1dYYP4NX3jyN08AOc+QX6xO1H74nkVWuQKhonGo+gx8Upj7YO+HkE/B7WThvJ0ave7/NyBLiJBezjQO7gYjyuEG93TOYA3x5Hh6LGL3+QlmsWEPZ4Ob8zxJdZwh+J1swmZkOFg8nvK1amlGIfFdM9drDTd+NLQ/wmny3F7lVktb5c12RFvl1ao9eXvM1LkGv+9QvumTl8v7jtgvtOOrAoYbJEr8cbimT9eewZ7ifkTt4mwC18nRG00Ruq4fpnfsUPnriTi5Y9ztPvfizpnOXsGNu5YIWlN1HT0U7LNQvw9PZQ07mXOrq5k3mMq3sfX02kLxsqns3kq4lQWx9Jek0ZvKhH4RAytWmA3G/yxe5VVKomefl6PgG/h0c+eCynvvYaQbx4CfJlluBxhTl410Q8pgMTMXhEmOAbDj0FLbOf17N22sisP4+uei8uI+zvYxS7BrwcJhtYJS0Ewn4CYT+wPxQ1eVtrWbOlMjX8q9+6iYjXC737P0h3nYerv/MKoROPSjIEmbKZlMGJGgoH8cYHh/dV3VppFLne5IvZq6hUTfIKMUBd4yKcu/aPBE0Na5jG6TzG+tCReB4PcPp6w30nHRg1EsDS4e05T8aLYxV2m/LK9j6Be6DPI+D38NoxyeEngFpXN+ccfyuvvjSVUGR/23SPK8TejhFMffGFkoT68qFz9DhcwWRvyR0O0nDiwfRaGAKrbKaajnbqt26ic/Q42mhSQzKIUEPhABKfVo0xvDFpOBsPawAKy3qC4hZnlaJJXr4GKPqZ/YtPuM7FhN38l9zAT8x3qaMbQnDSRph53z5umTukr4FgPpPxIL3Xs6ehlsfOOjSrz2Pj4Q0ghimrYwV7Bv7ZMpJRB7X2q34ORTw001rWeRgDtQ/vbWhk5dU30rLwCiIeL65QkJVX30hvQ2PG98WJ6xtRryTElyN38OfauTrJbpCghsJmrJ5Wj3h9Z99QGrtTUFMpxZpyNUCJn5knVi/xC/kmERdJTYKMS1jv6+xrIBiKTcbL1VBk8npy+Tw2HjacbWOHJF3nUGLVzyuvw+MKEQx7OO+Dt7Gvrqaoob6BssqyKZDbOPts3m85sc8ryNZIJOob8dDVYi7i0X1nsIMmTZ8dBDjrLlSFOG2Sml3kcsO1/swMLosb6+RAPR6zmxBRj2JKV11eaytW2M3qOk+esJyjD3qBR978DEtfv4QHNlzEn16/hMAhn2bu248UfM6BsspWTHmWxXOyK5DrbWjM2kDEsdI3gnhpppUdNGn67CDA7sFFs4FfAm7gDmPMDSmvfwn4KfvHny4yxtxR1kWWGCdNUisXhRbtBT0uXOGUz8zA2ukj+2kGh0YO7GsgmK9GAeWZTbH09YuTRO15b9+P78yTGRVqy/ucVh7r0S9s5+1hY/AP3cd7Fz3O7R9rIBjIXCCXqC/kaiis9A0vQVppBjR9djCQ8ZsnIkOAJmPMWynbjzLGvFrIiUXEDfwKOAPYDLwkIsuMMa+n7PpHY8xlhZzLyThlklq5Kq5zrZlIXVff+zEYIOyOysPx46SGdmB/A8E4G/zdeRmOUoYCrVp6e1whNoWa8Tfuy/u4Vt5XV6SeXzxyE1On/41//fpAgoHkNN/UJ/xEfcEVjGoTG2efnfUaUvUNAiHmR+6g09+IL1T69NlCjJwSJe23XkQ+A/wC2C4iXuBLxpiXYi//FjimwHPPAN40xrwdO999wNlAqqGoeOyepFaupnW5Fu31S0mNZRl5Er0JY3hq9kQ6h9YAA9/MN/i7+f64jQWJ26XAqqV3KOJhZP2WNO/IDiuP1UuQNyKTeP6lE+ElSB0eFAruf8K30hdaFl7B+y0n5nTTTdU3TqKJSVvbSp71VKiRU6JkKrj7HjDdGDOV6Kzqe0TkU7HXijFuagywKeH3zbFtqXxaRF4VkaUiMi7dwURkvoisEpFVe8pUnFRMAn4PuxprbfEk8il4y4dcivas1jVl9XZMyjfPuF14Q9nfaNbWde0XtyUqbheb3T0NvNE+hd09DVm/J97S2+fups67F5+7m6+1XFVwlXbA7+HeiR+ji1p2M4QuavkyS9hBE9E/49Q/ZcMnL9rTd/Pu0xcSCLu9dDz7Hns6cqvX7W1oZOeRU+ltaGRIQ4RDjwyW3JOIGznfvr14entoWXgFNR3tJTtnpZLpruQ2xrwHYIx5UUQ+Avxf7GZdrgG2fwHuNcb0isjFwF3AqVY7GmMWA4sBDmus1QG7WVJOMd3q6dYVjjbWy3ZdrpT7Sq56TuJ0vHzF7UwkzZjOseFfXNTe3jmmXyuPfEODu3sauOidpXyLfTTTSivNMSNhjbfG9M2EAGt9IdwVYuFPpvH+9U2OTm21EtEjHi/1WzdpCCpHMj0S7BWRQ+O/xIzGLKLhoSOLcO4tQKKHMJb9onX8nO3GmHh7sTuA6UU4r5JAOcX0uB4Tcgsht2AAg+GUR1sZ/e7ugddl4LVjRhJyC0Gvi5BbMuo5Vu0v4tPxPrejiWs3jQeixXgb/IXf7BJnTHcFDyQQruWWldfl7Fkc3rg2yUiMbt3N6cveYuYTmzh92Vv9PqtMxLWPHTSxiuNiRsLQ/1nP4K2JcHGKXhDXF0I1fnrrDuzzSDZ1HkSg18XihQ05exblwsrIuUJBOkenDUwoacj0aHIJKX6pMWZvLFPpyiKc+yXgcBGZSNRAzAUuSNxBRA6OezXAWcD6Ipy3rDipLbcV5RbTt04Yyu5hfk555B0E8EQATD+tIt26tk4YyrZxBw74mWbSXeLidrH1inSC9PbOMXmHkNLpOruH+fGGIgN+r6y0D68PzvvP3Sy9dQhuT1ST+ORFezjt012WoaCNs8/mzQ+czNZHt/Hr3x3F5q7+XWOdmNpaaJGgsp9Md4MHgdtE5GfGmDCAiBwE/AyYBFxTyImNMSERuQx4lGh67J3GmHUicg2wyhizDLhcRM4CQsBO4EuFnLPcDJbJZuUW072hCMbtgsj+m4tVuCvdugYSrLMVzZP0ijyL8RIphSBtFYIDOOWRdzDugVvQ940zXXkdbn9N3zjTj7Rs5tOHb6GVCdRPash4o48OIhqNy3MkPV3JmobTU1vzLRJUksl0R5gO/BhYIyJfBz4EfAv4CfDFYpzcGPMw8HDKth8k/HwlxfFeyk4p23KXgnJWgecS7spnXdnqLsXWKxJvyokaRSGCtNVn5Q6bqKsfya4FfVz7ePecB/DXGQ55/AHOWvitATOB9nS4aN3g5faFyXUWYPDXGSLh4s3QLiX5FAkqyaT9CzTGdACXxIzEY8BW4HhjzOZ071H2oxXX6Sl1uCtbQxTXKwotxkskkyCdD6mflSscrSDx5DiSteuypbx6fw3P3tHLm4Fv4yFzumt8nKm46Fdn4a8zXPidDqadqG03qoVMdRTDgP8GWoDZwBxguYh83Rjz9/Isb/BSjRXXuVDKcFcuhmigYrx8NKah/o6iDh+yGpiUKEYP9L16qnUOt3xsFMGAcCyrCOCLNk+MkZoJlDTq1IJIGDUSVUamb/7LwK+BrxpjQsDfRGQq8GsRedcYc345FjhYcUrFtZPJJay0u6chp6f0fAxRqrh964vD+eKKnY7QmBI/q1y+V7t7Glj08k+joSOglWZ8sUaKcVIzgdq2uvuNMwVDjd9gTPpwUzaNBZXBSaa/npNTw0zGmDXACSLylZKuqkKwu+LaLoqd6ZVvbUKu+kaquP1eVzueMI7TmHL5Xr17zgO4V9B3099BE19mCXcyjyBe6msC/TKBmkaH+40z9dYYvvmzHTRPClkagXioKpvGgsrgI5NGkVaLMMb8T2mWU3k4sVV4KSl2pldibUI87TQ+Aa7Ys6UTxW1vRDh5IySGeJykMSV+rzIZZqub/h+Zy9O+U/nmRa9y6KdH9hN64+NMFy9swO2hL1PqqJnJnkicpFBVhsaCyuDF/m+8UjGUItOrFLUJ6UgUt6ft8jFz01aeHwtPNsOsVjjuPedpTJkMc+eCFQwh+aa/v2YizJCGycnRpQRyGWdqFapycn2FkjtqKJSiUYpMr1I1y0tHorh990eHc9kx7QTc4AvDopeHM8IB3kScdIZ547XLk7yEfGdYW40ztcLKa3F6fYWSG86svVccS6aGd6XI9CpVs7xsePIQF70eCLug1xP93UlYNVmM+A+gfuumfvuWsglfPFTlq4lQWx/BV1P61uFKeXHO45HieAYSlUuV6VXs2oRsKXUDQStyye6ybLIYy2DKJgOpmFlK+XotyuBAjKm8RquHNdaam85stnsZFcXungYuWvY4gfD+mgOfu5s7zjqt3w3N6f2tciHfIUf58Mgb53HHy1fhcQUJG3dW2V2j39293zC7fKy8+kZ+s+987vrpMDw+iISsM5A0S6kyOX/a2LzfKyKrjTHHWr02uP+KlbKRi6icKdMr13qIgSj28VJJ1CziRqN510EMbZtU1HMuf+M8blu1EBCCkegQpltWXsfEYRvoCdUnnSvxmplAX6rsju88xF8fH8ed1zcAQiimG6RmIGWbpaR1EUocNRRKVhRDVC5kVkM5jpeJeDFeEDDD9lHzj59jNn24KOfc3dPAHS9/j9QhQiYifPORB/C6A33XZ4xYXnPH959gT4eLu3/a0O84LndyBlI2WUrqcSiJOEudUxxLoaJyMWY1lPJ4A7G2rougy4VxGXAF6B37Uk7ntJqNEWd75xi8rv7bg6aGYKQm4fqu55aV/a/5vYseB+Dhe+r7vIhEEkebwsBZSokeR/c+l+PnTiilRz0KJWsKEZWLXQ9RzvqKzgUrOHTnatz/mEsoBER80Dor63MOVIQ4sn4LYeNOeZfB5+olEPH3bRHCSMxZGEEbzbSyXUbQtrWOFx/3s+y3Q7AabXrhgl1JoaN0BXW5eBxKdaGGQsmJfBveFbseotT1FZ0LViT9fsTw6XzrqD/ys1+uJfzmR2DzTBj7PO6JD4L/Zep3G8tBQtkUISbNjJAwwYiHzx/1c/7w2jeS1mBwY4xhLveyhHkE8OELBXh89c+45tf95owBhs/PfZvzJq+ns2Nc1rUVWhehpKJZT0rZePrdj/Wb1VBIfL/Yx4P+BiKV5x6pZfHCBhj/POHPnAruAL4wPHYXzHgveqNO9BiGtXcz84lNeIP7b8RBr4vnPzKOXY3JWVSpwrxVFlR9sIvvvHR3UvfXoM/PIZ7WpMlzABfI77nLd9GAcycyXWeix6Hpr86nIrOeYmNVf0l0wt0dxpgbUl6vAe4mOkSpHfisMaa13OtU+pNPtlGx6yGKfTwrI1HT0Z40HS3+JL7s5T/x6N4AYRcEDDzVDCds7u8x5FKEmOitPdU6hyWvXInHFSQY8fCV6ddx8oTlDGvvxuMJRGc+xoh4vIwLtrKZ/YZiBNu50/0VPL2Z506kI9XjWPtCDZfPGaXidpVim6EQETfwK+AMYDPwkogsM8a8nrDbPKDDGHOYiMwlOh/js+VfrZJIIdlGxZ7VUOjxMhmHYevXMv1nC/s9kQ9piHD6sLE8sStqJHyRaC+oOIltS/IpQkwU6uMsefl7zBz7OL76EC63F0L7w0DucJBTFgxj9c8iuNwQDgmXf+5V5E8e2Lf/uKlzJwYi3sJDm/4pdnoUM4A3jTFvA4jIfcDZQKKhOBv4YeznpcAiERFTifEyB5GpYC61m+sI2njuhU8xY9gz+IfuS3NEZ7G7p4F3z3kgGkIh+UY3fvmDtFyzgIjHjbezMxr1T3kiB5gaOJhH/+Dl2TFBZrXCzM30NRA8cWMkyWNIbAu+zdPEplAzI3vSe0AZhfrGDlZefSMtC68g4vHiCkUN2FGz67n5tG19HkATTbj+kCw0pM6dyBYVtxU7DcUYILEpzWai0/Qs9zHGhERkN9AI7Eg9mIjMB+YDNNWpRp8vA2XoJN7E4qJqMOKl7pFO/nn8yJxbipe6YC6VFVOejdYHrOgfQqnpaKflmgWxcE3/90Y8Xg5d+juOvHMREa8Xdy+csNGDcbt5bmwvp18IATd4DFyzOciknv3fw4Dfw4ptZ2XliWUS6jsXrKATeL/lxKSQGCQ38eul0dKg5DM7WsVtpWLuqMaYxcBiiIrZNi9nUJJNhk78JjaCNpYwLyaqdkMk95bi5SyY61ywIhpCmZM+hFK/dRMRr7fPg0jFFQpw5JJb8AR6+/YJ1dTw9E1LuMv1JL3v30mECMZE6y4SW37kMlcjMQsq8bMZ6u+gM7ZPb0Njv5t+qp6ycfbZlgYlVwZKp1UqHzsNxRYg0Q8eG9tmtc9mEfEAQ4mK2koCxXoqz6ZNePwm9twLnyIY8QLJguaQjh52HHxAVmvO9sYZD4UFPS7LFNR0pOoPA4VQOkePwxVMfnSOfxIRr5d1877G5Ltvh8D+A0Q8PoJDhnLIwR/H0/Y7QuEeywaCudZ9WAn1qdeT2GJjygt/jobMUvQUK4OSK3s6XBw0LsR1f9hOT5do1lMVYqeheAk4XEQmEjUIc4ELUvZZBlwIPA+cC/xd9YlkivlUnm2GzskTljNj2DPUPdJJYojfHTbMeHoza44/eMAQVLY3zngoDGNwRyDs7p+Cmkq6FNeBQii9DbFwzQ+/jTvQi7C/MsGIi02nzeHIJYuS3h+P+x/R0Mj3TriP9e3Pc8wj9/VrIOj3dBIM+5K25VL3kXpNiS02GgJtvGP+C08wvwynTFi18jj0SIvyb6Wisa0m3xgTAi4DHgXWA/9rjFknIteIyFmx3ZYAjSLyJvAt4Lv2rNaZFLuNRTxDJ+QWgl4XIbekzdDxD93HP48fSdgtfU/dAnhiISirVhWJZFMwlxgK80Rixw8bPGFjeY7OBSvoXLCCmo52hq9bQ01HsvOZzdyEjbPP5pmfLyFUm+wRRLw+vF2drLz6RkI1fgL1BxKq8SfF/Y8YPp2zD7+McV97lg3+bpYOb2eDv5unWufwrUfvJ2pVDT5394AtUJ5qncNFyx7nB0/cyUXLHue5R/YbntQWG6MDG+kOJhvzeIZTIWgrDyWOrRqFMeZh4OGUbT9I+LkHOK/c6xoslKKNRWKGzkAhnq0ThhL0uTnumS1RXSNGNlPtMsXh41iFwlLP0fH9J5K292UtpSkys6pITo3td0yagkSSQytxz2HnkVMHjPv/e+dqrh+3kZAYPMZF5PFPEEpIdY0Y4Rezz2Hc0HcsPxursNzihTV9WkpqCK2VZrwUJ8MpEc12UuJUjJhdjWT7VJ7rbIhMbcJT2d3g77ct26l2AxXMWYXC+s7h8rHjOw8lbUvOWkofgknMDkpnWDJlDA0U91/f/jwhl4sIYYImgmfik7BxVt/rPneQnlB92vdbPQAk3qBTQ2g7aGK++w5+65lXcIZTIprtpMRRQzGIGeipfKBU12JQ6FS7TAVzicfu0yhqoobJ6kZolbWUqcgsk2EpJGNocuNMPC4voQi4XF7MO7OSXh9Im7B6AEi8QVtlITVcfSbLWl4oOMMpEc12UuKooRjkpHsqzybVNZFCptLlEq7Kla0ThrLx2uXUb91EsK4eb1dn2huhVdZSphDMQIYl1XPIdpDPEcOn8wl5gAeffBnXllMIb5mJe/zTuCb+HfPOLL42dlnG0OBQfwfzr+1i8cKafjfoeJjsIy3jmPJwcgitl8IznFLREacKqKGoCKyeyrNJdY1TDM8jl3BVtiRm+mRzA+zLWsqyyCwXwxLP/nF5IBSACxfs4rRzuyyPu6fDxV9+9FHCvbMJA4x9Hr4wm4i7B+8p1zJy0zgYYKyq1Q16IP2lVCSG6pTqRA1FhZJtqmuunkc5GKiDayZyCRllY1j2dLho3eDl9oUNBAP7C/WWXN+AETj90/2NRT8RuPlJcAcwLkMw0r8YL5X49SfeoLPVXxSlFKihqFCy1Q5y8TxKTSEGIpFciswyGZa4FyEuCAZSZz0I9/y0gRmn9vR72u4nArfOgrAPTAAT8dG86yCS2r9mQa76i1JdxEOSjK2BpqaiH18NRQWTjXaQSxvsfBhI+yiWcSiEVMNS09EOG7bw54XTCQTS1wy4PcYyVTQuAt92dQOhoESHHN31ODQ/Sc3m4xh6xO3QuNbymOk+j1z1F6V6SAxJEg7BkiVw/vlFPYdWzlQ4Ab+HXY21ab2DXIrscmV0625OX/YWM5/YxOnL3mL0u7v7XosXxzmN8csf5Kw5xzPnirm8EZjIZ7k34dVkgxoJp08VPWF2Nz++bzveeDH25pnw7JWY905ld9OGvmK8bImHydIV+1UT6Qoqq5HEkKRv317o7oZ586Ctrajn0Ql3ClBY1lO6452+7K2kQryQW1j2t1cce3Or6WjnrDnHR3WAGF3UMoF32UETLncEl0vweA2RcHbDe1InxX3i+3/jL5FP9fWEunbT+D69IhvDmVocWG3YJejbQTb/1sPXreEjl14QNRJxhgyBxx6D447L6XyOnXCnOIdiZy1ZaR8R/wGOjqlb6QBBvDTzDjvdjXz64j3MOK0np8Z4qdlLT+x4ktD6IBGBEKZP2M7WuypGk7/BSjUJ+tkaRKuQJMEgNDcXdT0aelJKQle9F3ElN8Fzakx9T4eLt9Z52VY3od8fnZcgrUwkEhb+ctcQrrpgJO9v8uSULjqkIcKhRwYZ0hDpK8Zz4cbj9vfrMqukp8+QJ1CMnlZOIzWc5OntoWXhFZahttSQJLW1UY2iyIK2ehRK0YkP11l53ENFGZxTSpK7ozYR+eTNfP6By+kK+PAS5MssYQfRP7qezmjmU+oY0MRCPCBjcdoRw6f3dZmd3DiTcR+fzis7V7P+jUVMbpzJEcOnl+nKBx/VIujnmuGWmLl35pktmvWkOBer0EmxBueUCqtZ0Jc8OI+u22by+4s7eSN4SJ+RSCSx71KioQn2CiZi8NX2n56XyBHDp/cZhH/vXM31z80lFAnicXn53gn3qbFIQ64FlYOVfAxiX0iyBEYC1FAoBTJQbN3JMXWr7qgisLl7JNMXulm9sAG/O0JPV+Jkiv19l6wMDQjdsdHhqZ6HFevbnycUCRIhTCgS/V0NRXqc/vBRDJxoENVQKHmRadraYGn3YNUdtbdH+Nm3RvDFb+/iWze1A4a2LR7u+dmwfn2X3lrn7WdoEsnUkjv+eU044IS+BoIel5fJjTOLfp2VhpMfPoqF0wyiLYZCRIYDfwSagVbgM8aYfl3SRCQMvBb7daMx5qzUfZTyYuVBWE1BGyhtFLI3LjUd7QzbsBYBOiZNKdofTbww7vYfNsQqr6P/BXuFJdc34K83RGKG4eaHt/Vbq5WhSSRdS+7kz+vjfOL7D+D5wJOqUShJOMkg2lJHISI/AXYaY24Qke8CDcaY71jst88YM/AA5hS0jqK4ZAov7elwcfmcUdHwSwxfTYSbH96W8eafrXEZv/xBjv/BN3GFoy0vIl4vLyz8eV+qYDE8mVefr+HnVzTS222dBJjpehLrJIKBmEbh3+95pF5TNp/Xv3eu7hO71XAouXD+tLF5v9eJdRRnA7NiP98FPAn0MxSKvWST25/PFDSr2L5VPL+mo52WhVfgDu/vi+QOBmlZ+G3ebzmRJ14Ym5cnk0rzpCAmg43JdD2pdRKQOetpoM+rnOJ2YkEX4Jgwh+I87DIUBxlj3ov9vA04KM1+fhFZRbSD2g3GmAfTHVBE5gPzAZrqVHophFxaa+QzBS1b41K/dRPG7e73fuNyw4YtLL7mQwMam2xIHNDjcpNWvM70/sRzZjr/QJ9XKcRtK68rsaDL3d0NLiFc48+q2rnaq8OrkZLdUUXkMWCUxUtXJf5ijDEiki7+NcEYs0VEDgH+LiKvGWPestrRGLMYWAzR0FMBS69K8u27lM8UtGyNS+focUi4/w1aImFamVDUec6JnsE7G7z87sb+4nUxGOjzSpyOVwxx2yrE95GWzf0qnAlHvTXIXO1cTS00lP3YpVH8C5hljHlPRA4GnjTGfGCA9/wW+D9jzNKBjq8aRfYUqzFfrlpBag+ktBrFIw9x/Pe/0U+jWNvyqby0kVyvx19ncmrZkevxrY6bqFEAeesV6fSQe296hI9/Z25yf6AEgrV1PHvjYrbNPCVpu1UvrFCNn2UPv6CehUOoNI1iGXAhcEPs/w+l7iAiDUCXMaZXREYAHwZ+UtZVVjDF7tya6xS0bEdsxtMEfS+9zu6dQqjlSHwTGxhC4fOcM92shzREWPtCTVE0ECsyfV7xgrxC9Yp0Ib5W+rcqScTT3cVJ35zHyh/+LMlb0JkY1YtdhuIG4H9FZB7wLvAZABE5FrjEGHMRMBm4XUQiRHtS3WCMeT2bg3tCEXw9IdsmtDkVp7X1zta4REXrD0Vv2L/cf8MuZJ7zQFlXezpc3H5NA8EiaCD5UqhekS7EVz+pIamgy93TDSK4QkHi6own0NsvBFUtLTSU/thyJzXGtAOnWWxfBVwU+/k54EP5HL9+b4DTl72V1+znSsRpBiIXBsqQsjI2A4XB0h1zwgeCfWGmx5fWEexNnmpXiAaSD4XqFZn0kNSCrmEb1nLSFfPxdu8f7ZrqLTixYlgpDxX5yC0GPGFj++xnJzCYjQTknn6bTX2G1TEBrpw7Em8NhAIQiSRnPgGEBsh+KjapDQTzyX7K5HUlFnTtmjQFiSR/nlbegtMqhpXyUNF3ULtmP9uFk41CvoVxuaTfZlufYXXMQG/UMIT6tqcmeRg+OW/PgGsvdiuTxAaCkF8xXjYhvly8BSdVDCvloaLvoMWc/exknGwgIP8WH5Bb+m223kfqMUNBATH9Qk2pHDjAzbaQ68yGUhfjqbegpKMiDYURCLmKN/vZqTjdQED2T/mZyFa0zsX7SDymv85w1QUjk153uQzRSEzceAi/u3EYM07tyUn3KKb4XY5Os+otKFZU5IS7zgN9PHbWoRUpZHcuWNH332Cg7yk/gfhTPuyfLrenI/NXMXFKXKZ95l/dga8mQm19BF9NJGPKbPyYYyaG+r3v05fswV+fHH5KXHeu11kMkqbjaadZpYxU5ON2yOOqOE9isBiGVDI95ZciVJNvyqxVz6aHlgyxXLcV+bQyyZVUcRvgIZ2Mp5SBivQossHXE2JYeze+ntDAO9vIYPIerEj3lA/0hWq697kI9LpYvLBhQM8i23M2jQ7TttWd0/ESvZZ8vJNc9reipqOd4evWWM5GjnPE8OmcffhlAFz/3Fz+tP5Grn9uLv/euTrr8yhKrlTWY3eWjG7dzdQXt2FcgkSMI+stBrNxSMXqKd9q6E+x6hSK5ank6p0UUgCYaw8lnYynlJOqMxS+nhBTX9yGJ2wgHI1BO6XeopKMQyqpKZqlCtUUW1TOtTVJrvtDrJ16SpO+TI35oPjNAxUlE1VnKOo6gxiX9BkJsL/eopINRDry6TqbDfnMx7CbfHooFaMYT1GypeoMRVe9F4kkZ7PYVW9RjQYikUJCNWBd3FYOUbnY5NtDqRjFeIqSDVVnKAJ+D2taRjF1ZbJGUS5votqNQyr5hGqgvw7x+St2MXFSkKbR4ZJ4KqWkGD2UyjkZT6k+qs5QAGydMJQdB9VT1xmkq95bFiOhBqJ4WOkQd17XgL/eEIkZhpsf3lbUVhqlptCqaBW3lVJSlYYCop6FGojBiXVTP6GnM1pFvXhhAzc/vI1Dj0w/c8GJFFIVreK2Ukqq1lCUEjUOpcVKh0jE6eJ1KbASt1WzUIqFGooiogaiPCRmTLnc0NOV3BI8GAB/XfWNTU8Ut1WzUIqJLZXZInKeiKwTkUhsql26/WaLyL9E5E0R+W4515gLg716OhuyqRouNYl9oU6Y3c3ND2/jqtvbmPe9aEW01xcBDC4XXHXBSJ57pNa2tdpNsmYRZH3783YvSRnE2OVRrAXOAW5Pt4OIuIFfAWcAm4GXRGRZtuNQB8LXEypYzK504xAn16rhUpCu2jredmPS9ADfO/8gQGKzJco/utRJqGahFBO7RqGuBxDJ2P9/BvCmMebt2L73AWcDBRuKQlp4VKJxyDRsJ5+q4VKsb6Bq654uweMzBAP7v1PVqFXE0YI8pZg4WaMYA2xK+H0z0JJuZxGZD8wHaKpLf1n5tvCoRAMBA/dFyqdquNhkU209GAvtSk2qZqFGQ8mXkhkKEXkMGGXx0lXGmIeKfT5jzGJgMcBhjbVplcxcW3hUqoGA7J7U860aLibZGIFStQSpBFTYVgqlZIbCGHN6gYfYAiTejcbGthVENi08Ktk4JJLNk3oxqoYLJVsjUGhLkEpFi/GUQnFy6Okl4HARmUjUQMwFLij0oJlaeFSLgYiTbbjGqmo4k65RCrI1Avm2BKlkVNhWCsUWQyEinwJuAZqAv4rIGmPMmSIyGrjDGDPHGBMSkcuARwE3cKcxZl0xzp/awqPj+08U47CDjlzCNYlVw8WcTFfT0Z512wo1AvmhxXhKoYgxlVeYdFhjrbnpzOaM+1Sb95CJXLyDPR0uLp8zKqprxPDVRLj54W0538SdkHZbjahmUbmcP21s3u8VkdXGGMu6NieHnkqCGoj+5PKkns+8BytD5IS022pFNQslV6rGUKiBKA65pqGmC1M5Ie22WlHNQsmVijcUaiCKSy66Rqb02xoHpN1WK1qMp+RKRRqKyEGH07ngYbuXUbFkm4GUMUx1pP1pt9WMFuMpuVCRhsIplDuFtJxko2sMFKYqdFiPUjgqbCvZYEv32GrgueW1XD5nFNdf2sTlc0ZVZSfTeJjKVxOhtj6CrybSL0zV29DIziOnqpGwCe0yq2SDehQlIJvWGOVci51eTWqYCuCtdd6K9LIGIypsK9mghqIE5JNCWgqKWRhXCPEwVSHrsdvgVSpajKdkgxqKElBIJ9Ni3RCd5NUUuh6nGLxKRSfjKQOhGkUJyCY2b0UxdY0+ryaBuFdjB/muJ9HAdO9zEeh1sXhhA3s69KtbClSzUKxQj6JE5NrJtNgegNPmM+S7HqeE8aoF1SwUK/SxrITEx3Rmc0MrtgeQr1dTKvJdT7kNXuJc7mokrlmcN/kKDTspfahH4RBKcUN02nyGfNZTzoFEqoVESdQsQAvyFDUUjqFUN0SntebOZz3lMHhOE/+dgorbCqihcBRO8wCcRKkNnmoh1minWQVs0ihE5DwRWSciERGx7H8e269VRF4TkTUisqqca7SLXHQNpXg4Tfx3CnFx24Vbxe0qxi7Fbi1wDvB0Fvt+xBgzNd1ADUUpBk4T/51CqrgN8NAbi/j3ztU2r0wpJ7aEnowx6wFExI7TK4olGvqzJi5uq15RvTg9B9AAfxOR1SIyP9OOIjJfRFaJyKq9HTvLtDyl0tDQX3q0GK96KZlHISKPAaMsXrrKGPNQloc50RizRURGAitEZIMxxjJcZYxZDCwGOOSDR1XeIPAyoT2VlHRoMV71UjJDYYw5vQjH2BL7/3YReQCYQXa6hpIHlVZHoEavuGgDwerFsemxIlIPuIwxe2M/fxS4xuZlVSyVVkdQaUbPKWgDwerErvTYT4nIZmAm8FcReTS2fbSIxGeYHgQ8KyL/BF4E/mqMecSO9VYDTmsiWAh7Olzcro0ES45qFtWDXVlPDwAPWGzfCsyJ/fw2cHSZl1a1VFIdweNL6wj2JmfUafFc8VHNonpwbOhJKS/l7KlUSvZ0uHhwyRAg2VCEBqnRczJWmoVSmaihUPqohDqCtq1uPD4IBhK3Gj45b8+gvB6now0EqwM1FEoSTmsimCtWITSvz3Dap7vsWVAVoeJ25aLqnuJY8pkNYdWK4+IfDr4Q2mBExe3KRT0KxZEUkt5aCSG0wYiK25WLGgrFcRSjpmOwh9AGIypuVy5iTOV1uxCRNuBdu9dRACOAHXYvosRkuMYD6uDwI8CVUMQRCcMb/4Z9g01sqPJ/y4qi0q9zgjGmyeqFijQUgx0RWVXpbdWr4RqhOq6zGq4Rquc6rVAxW1EURcmIGgpFURQlI2oonMliuxdQBqrhGqE6rrMarhGq5zr7oRqFoiiKkhH1KBRFUZSMqKFQFEVRMqKGwoGIyE9FZIOIvCoiD4jIMLvXVApE5DwRWSciERGpqLRDEZktIv8SkTdF5Lt2r6cUiMidIrJdRNbavZZSISLjROQJEXk99l39ut1rsgM1FM5kBTDFGHMU8G/gSpvXUyrWAudQYeNtRcQN/Ar4GPBB4HwR+aC9qyoJvwVm272IEhMCvm2M+SBwPPDVCv23zIgaCgdijPmbMSYU+/UFYKyd6ykVxpj1xph/2b2OEjADeNMY87YxJgDcB5xt85qKjjHmaWCn3esoJcaY94wxL8d+3gusB8bYu6ryo4bC+XwZWG73IpScGANsSvh9M1V4c6k0RKQZmAastHkpZUebAtqEiDwGjLJ46SpjzEOxfa4i6vr+vpxrKybZXKeiOB0ROQC4H/iGMWaP3espN2oobMIYc3qm10XkS8DHgdPMIC52Geg6K5QtwLiE38fGtimDEBHxEjUSvzfG/Nnu9diBhp4ciIjMBv4LOMsYM9i6pSrwEnC4iEwUER8wF1hm85qUPBARAZYA640xN9m9HrtQQ+FMFgEHAitEZI2I3Gb3gkqBiHxKRDYDM4G/isijdq+pGMQSES4DHiUqfv6vMWadvasqPiJyL/A88AER2Swi8+xeUwn4MPAF4NTY3+IaEZlj96LKjbbwUBRFUTKiHoWiKIqSETUUiqIoSkbUUCiKoigZUUOhKIqiZEQNhaIoipIRNRSKUgJiXUffEZHhsd8bYr83i8gjIrJLRP7P7nUqSjaooVCUEmCM2QTcCtwQ23QDsNgY0wr8lGhuvqIMCtRQKErp+DlwvIh8AzgRuBHAGPM4sNfGdSlKTmivJ0UpEcaYoIgsAB4BPmqMCdq9JkXJB/UoFKW0fAx4D5hi90IUJV/UUChKiRCRqcAZRCejfVNEDrZ3RYqSH2ooFKUExLqO3kp0fsFGogL2jfauSlHyQw2FopSGrwAbjTErYr//GpgsIqeIyDPAn4DTYl1Xz7RtlYqSBdo9VlEURcmIehSKoihKRtRQKIqiKBlRQ6EoiqJkRA2FoiiKkhE1FIqiKEpG1FAoiqIoGVFDoSiKomTk/wcRbytLrgQKGgAAAABJRU5ErkJggg==\n", "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + "
" ] }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - " custom_fft_abs(x, verbose=1, fLOG=print)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "22.6 \u00b5s \u00b1 6.06 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit custom_fft_abs_py(x)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "269 \u00b5s \u00b1 4.46 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" - ] - } - ], - "source": [ - "%timeit custom_fft_abs(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Again the gap is less on bigger matrices. It cannot be faster with the default runtime as it is also using *numpy*. That's another story with *onnxruntime* (see below)." - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.68 ms \u00b1 55.9 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" - ] - } - ], - "source": [ - "bigx = numpy.random.randn(10000, x.shape[1]).astype(numpy.float32)\n", - "%timeit custom_fft_abs_py(bigx)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.35 ms \u00b1 76.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "%timeit custom_fft_abs(bigx)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Using onnxruntime\n", - "\n", - "The python runtime is using numpy but is usually quite slow as the runtime needs to go through the graph structure.\n", - "*onnxruntime* is faster." - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" - ] + "metadata": { + "needs_background": "light" }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "@onnxnumpy_np(runtime='onnxruntime')\n", - "def custom_fft_abs_ort(x: NDArray[Any, numpy.float32],\n", - " ) -> NDArray[Any, numpy.float32]:\n", - " \"onnx fft + abs\"\n", - " return _custom_fft_abs(x)\n", + "import matplotlib.pyplot as plt\n", "\n", + "def draw_line(ax, v, p0, rect, N=50, label=None, color=\"black\"):\n", + " x1, x2, y1, y2 = rect\n", + " v = v / numpy.linalg.norm(v) * (x2 - x1)\n", + " points = [p0 + v * ((i * 2. / N - 2) + (x1 - p0[0]) / v[0]) for i in range(0, N * 4 + 1)]\n", + " arr = numpy.vstack(points)\n", + " arr = arr[arr[:, 0] >= x1]\n", + " arr = arr[arr[:, 0] <= x2]\n", + " arr = arr[arr[:, 1] >= y1]\n", + " arr = arr[arr[:, 1] <= y2]\n", + " ax.plot(arr[:, 0], arr[:, 1], '.', label=label, color=color)\n", "\n", - "custom_fft_abs(x)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "108 \u00b5s \u00b1 46.8 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit custom_fft_abs_ort(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "*onnxruntime* is faster than numpy in this case." - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "248 \u00b5s \u00b1 6.11 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" - ] - } - ], - "source": [ - "%timeit custom_fft_abs_ort(bigx)" + "def zones(ax, model, X):\n", + " r = (X[:, 0].min(), X[:, 0].max(), X[:, 1].min(), X[:, 1].max())\n", + " h = .02 # step size in the mesh\n", + " xx, yy = numpy.meshgrid(numpy.arange(r[0], r[1], h), numpy.arange(r[2], r[3], h))\n", + " Z = model.predict(numpy.c_[xx.ravel(), yy.ravel()])\n", + " Z = Z.reshape(xx.shape)\n", + " return ax.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)\n", + "\n", + "fig, ax = plt.subplots(1, 1)\n", + "zones(ax, model, X)\n", + "df[df.y == 0].plot.scatter(x=\"X1\", y=\"X2\", color=\"blue\", label=\"y=0\", ax=ax)\n", + "df[df.y == 1].plot.scatter(x=\"X1\", y=\"X2\", color=\"red\", label=\"y=1\", ax=ax);\n", + "rect = (df.X1.min(), df.X1.max(), df.X2.min(), df.X2.max())\n", + "draw_line(ax, model.centers_[1] - model.centers_[0], model.centers_[0],\n", + " rect, N=100, label=\"hyperplan\", color=\"green\")\n", + "ax.legend();" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Inside a FunctionTransformer\n", + "## Conversion to ONNX = second implementation\n", "\n", - "The conversion to ONNX fails if the python function is used." + "The conversion fails as expected because there is no registered converter for this new model." ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "FunctionTransformer is not supported unless the transform function is of type wrapped with onnxnumpy.\n" + "MissingShapeCalculator\n", + "---\n", + "Unable to find a shape calculator for type ''.\n", + "It usually means the pipeline being converted contains a\n", + "transformer or a predictor with no corresponding converter\n", + "implemented in sklearn-onnx. If the converted is implemented\n", + "in another library, you need to register\n", + "the converted so that it can be used by sklearn-onnx (function\n", + "update_registered_converter). If the model is not yet covered\n", + "by sklearn-onnx, you may raise an issue to\n", + "https://github.com/onnx/sklearn-onnx/issues\n", + "to get the converter implemented or even contribute to the\n", + "project. If the model is a custom model, a new converter must\n", + "be implemented. Examples can be found in the gallery.\n", + "\n" ] } ], "source": [ - "from mlprodict.onnx_conv import to_onnx\n", - "\n", - "tr = FunctionTransformer(custom_fft_abs_py)\n", - "tr.fit(x)\n", - "\n", + "from skl2onnx import to_onnx\n", + "one_row = X_train[:1].astype(numpy.float32)\n", "try:\n", - " onnx_model = to_onnx(tr, x)\n", + " to_onnx(model, one_row)\n", "except Exception as e:\n", + " print(e.__class__.__name__)\n", + " print(\"---\")\n", " print(e)" ] }, @@ -1042,50 +523,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now with the onnx version but before, the converter for FunctionTransformer needs to be overwritten to handle this functionality not available in [sklearn-onnx](https://github.com/onnx/sklearn-onnx). These version are automatically called in function [to_onnx](http://www.xavierdupre.fr/app/mlprodict/helpsphinx/mlprodict/onnx_conv/convert.html#mlprodict.onnx_conv.convert.to_onnx) from *mlprodict*." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "tr = FunctionTransformer(custom_fft_abs)\n", - "tr.fit(x)\n", - "\n", - "onnx_model = to_onnx(tr, x)" + "Writing a converter means implementing the prediction methods with ONNX operators. That's very similar to learning a new mathematical language even if this language is very close to *numpy*. Instead of having a second implementation of the predictions, why not having a single one based on ONNX? That way the conversion to ONNX would be obvious. Well do you know ONNX operators? Not really... Why not using then numpy functions implemented with ONNX operators? Ok! But how?" ] }, { - "cell_type": "code", - "execution_count": 32, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "from mlprodict.onnxrt import OnnxInference\n", - "\n", - "oinf = OnnxInference(onnx_model)\n", - "y_onx = oinf.run({'X': x})\n", - "y_onx['variable']" + "## A single implementation with ONNX operators" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [] diff --git a/_doc/notebooks/numpy_api_onnx2.ipynb b/_doc/notebooks/numpy_api_onnx2.ipynb new file mode 100644 index 000000000..bdc3995f3 --- /dev/null +++ b/_doc/notebooks/numpy_api_onnx2.ipynb @@ -0,0 +1,1115 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to a numpy API for ONNX: FunctionTransformer\n", + "\n", + "This notebook shows how to write python functions similar functions as numpy offers and get a function which can be converted into ONNX." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
run previous cell, wait for 2 seconds
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from jyquickhelper import add_notebook_menu\n", + "add_notebook_menu()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext mlprodict" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A pipeline with FunctionTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import load_iris\n", + "from sklearn.model_selection import train_test_split\n", + "data = load_iris()\n", + "X, y = data.data, data.target\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Pipeline(steps=[('functiontransformer',\n", + " FunctionTransformer(func=)),\n", + " ('standardscaler', StandardScaler()),\n", + " ('logisticregression', LogisticRegression())])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.preprocessing import FunctionTransformer, StandardScaler\n", + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "pipe = make_pipeline(\n", + " FunctionTransformer(numpy.log),\n", + " StandardScaler(),\n", + " LogisticRegression())\n", + "pipe.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's convert it into ONNX." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FunctionTransformer is not supported unless the transform function is None (= identity). You may raise an issue at https://github.com/onnx/sklearn-onnx/issues.\n" + ] + } + ], + "source": [ + "from mlprodict.onnx_conv import to_onnx\n", + "try:\n", + " onx = to_onnx(pipe, X_train.astype(numpy.float64))\n", + "except RuntimeError as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use ONNX instead of numpy\n", + "\n", + "The pipeline cannot be converter because the converter does not know how to convert the function (`numpy.log`) held by `FunctionTransformer` into ONNX. One way to avoid that is to replace it by a function `log` defined with *ONNX* operators and executed with an ONNX runtime." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Pipeline(steps=[('functiontransformer',\n", + " FunctionTransformer(func=)),\n", + " ('standardscaler', StandardScaler()),\n", + " ('logisticregression', LogisticRegression())])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import mlprodict.npy.numpy_onnx_pyrt as npnxrt\n", + "\n", + "pipe = make_pipeline(\n", + " FunctionTransformer(npnxrt.log),\n", + " StandardScaler(),\n", + " LogisticRegression())\n", + "pipe.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "onx = to_onnx(pipe, X_train.astype(numpy.float64), rewrite_ops=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%onnxview onx" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The operator `Log` is belongs to the graph. There is some overhead by using this function on small matrices. The gap is much less on big matrices." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4.43 \u00b5s \u00b1 311 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + ] + } + ], + "source": [ + "%timeit numpy.log(X_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16 \u00b5s \u00b1 2.13 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + ] + } + ], + "source": [ + "%timeit npnxrt.log(X_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Slightly more complex functions with a FunctionTransformer\n", + "\n", + "What about more complex functions? It is a bit more complicated too. The previous syntax does not work." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Pipeline(steps=[('functiontransformer',\n", + " FunctionTransformer(func=)),\n", + " ('standardscaler', StandardScaler()),\n", + " ('logisticregression', LogisticRegression())])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def custom_fct(x):\n", + " return npnxrt.log(x + 1)\n", + "\n", + "pipe = make_pipeline(\n", + " FunctionTransformer(custom_fct),\n", + " StandardScaler(),\n", + " LogisticRegression())\n", + "pipe.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FunctionTransformer is not supported unless the transform function is of type wrapped with onnxnumpy.\n" + ] + } + ], + "source": [ + "try:\n", + " onx = to_onnx(pipe, X_train.astype(numpy.float64), rewrite_ops=True)\n", + "except TypeError as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The syntax is different." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Pipeline(steps=[('functiontransformer',\n", + " FunctionTransformer(func=)),\n", + " ('standardscaler', StandardScaler()),\n", + " ('logisticregression', LogisticRegression())])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from typing import Any\n", + "from mlprodict.npy import onnxnumpy_default, NDArray\n", + "import mlprodict.npy.numpy_onnx_impl as npnx\n", + "\n", + "@onnxnumpy_default\n", + "def custom_fct(x: NDArray[(None, None), numpy.float64]) -> NDArray[(None, None), numpy.float64]:\n", + " return npnx.log(x + numpy.float64(1))\n", + "\n", + "pipe = make_pipeline(\n", + " FunctionTransformer(custom_fct),\n", + " StandardScaler(),\n", + " LogisticRegression())\n", + "pipe.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "onx = to_onnx(pipe, X_train.astype(numpy.float64), rewrite_ops=True)\n", + "%onnxview onx" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's compare the time to *numpy*." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6.34 \u00b5s \u00b1 522 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + ] + } + ], + "source": [ + "def custom_numpy_fct(x):\n", + " return numpy.log(x + numpy.float64(1))\n", + "\n", + "%timeit custom_numpy_fct(X_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "17.8 \u00b5s \u00b1 722 ns per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fct(X_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The new function is slower but the gap is much less on bigger matrices. The default ONNX runtime has a significant cost compare to the cost of a couple of operations on small matrices." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "349 \u00b5s \u00b1 21.4 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "bigx = numpy.random.rand(10000, X_train.shape[1])\n", + "%timeit custom_numpy_fct(bigx)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "367 \u00b5s \u00b1 50.1 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fct(bigx)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function transformer with FFT\n", + "\n", + "The following function is equivalent to the module of the output of a FFT transform. The matrix $M_{kn}$ is defined by $M_{kn}=(\\exp(-2i\\pi kn/N))_{kn}$. Complex features are then obtained by computing $MX$. Taking the module leads to real features: $\\sqrt{Re(MX)^2 + Im(MX)^2}$. That's what the following function does." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### numpy implementation" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", + " [2.7854056, 1.3178631, 2.491644 , 1.3178631],\n", + " [0.9064019, 4.1368184, 2.4568543, 4.1368184]], dtype=float32)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def custom_fft_abs_py(x):\n", + " \"onnx fft + abs python\"\n", + " # see https://jakevdp.github.io/blog/\n", + " # 2013/08/28/understanding-the-fft/\n", + " dim = x.shape[1]\n", + " n = numpy.arange(dim)\n", + " k = n.reshape((-1, 1)).astype(numpy.float64)\n", + " kn = k * n * (-numpy.pi * 2 / dim)\n", + " kn_cos = numpy.cos(kn)\n", + " kn_sin = numpy.sin(kn)\n", + " ekn = numpy.empty((2,) + kn.shape, dtype=x.dtype)\n", + " ekn[0, :, :] = kn_cos\n", + " ekn[1, :, :] = kn_sin\n", + " res = numpy.dot(ekn, x.T)\n", + " tr = res ** 2\n", + " mod = tr[0, :, :] + tr[1, :, :]\n", + " return numpy.sqrt(mod).T\n", + "\n", + "x = numpy.random.randn(3, 4).astype(numpy.float32)\n", + "custom_fft_abs_py(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ONNX implementation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This function cannot be exported into ONNX unless it is written with ONNX operators. This is where the numpy API for ONNX helps speeding up the process." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", + " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", + " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from mlprodict.npy import onnxnumpy_default, onnxnumpy_np, NDArray\n", + "import mlprodict.npy.numpy_onnx_impl as nxnp\n", + "\n", + "\n", + "def _custom_fft_abs(x):\n", + " dim = x.shape[1]\n", + " n = nxnp.arange(0, dim).astype(numpy.float32)\n", + " k = n.reshape((-1, 1))\n", + " kn = (k * (n * numpy.float32(-numpy.pi * 2))) / dim.astype(numpy.float32)\n", + " kn3 = nxnp.expand_dims(kn, 0)\n", + " kn_cos = nxnp.cos(kn3)\n", + " kn_sin = nxnp.sin(kn3)\n", + " ekn = nxnp.vstack(kn_cos, kn_sin)\n", + " res = nxnp.dot(ekn, x.T)\n", + " tr = res ** 2\n", + " mod = tr[0, :, :] + tr[1, :, :]\n", + " return nxnp.sqrt(mod).T\n", + "\n", + "\n", + "@onnxnumpy_default\n", + "def custom_fft_abs(x: NDArray[Any, numpy.float32],\n", + " ) -> NDArray[Any, numpy.float32]:\n", + " \"onnx fft + abs\"\n", + " return _custom_fft_abs(x)\n", + "\n", + "\n", + "custom_fft_abs(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`custom_fft_abs` is not a function a class holding an ONNX graph. A method `__call__` executes the ONNX graph with a python runtime." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%onnxview custom_fft_abs.compiled.onnx_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Every intermediate output can be logged." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-- OnnxInference: run 39 nodes\n", + "Onnx-Shape(x) -> Sh_shape0\n", + "+kr='Sh_shape0': (2,) (dtype=int64 min=3 max=4)\n", + "Onnx-Slice(Sh_shape0, Sl_Slicecst, Sl_Slicecst1, Sl_Slicecst2) -> Sl_output01\n", + "+kr='Sl_output01': (1,) (dtype=int64 min=4 max=4)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst\n", + "+kr='Sq_Squeezecst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Squeeze(Sl_output01, Sq_Squeezecst) -> Sq_squeezed01\n", + "+kr='Sq_squeezed01': () (dtype=int64 min=4 max=4)\n", + "Onnx-Identity(Sl_Slicecst2) -> Su_Subcst\n", + "+kr='Su_Subcst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Sub(Sq_squeezed01, Su_Subcst) -> Su_C0\n", + "+kr='Su_C0': (1,) (dtype=int64 min=4 max=4)\n", + "Onnx-ConstantOfShape(Su_C0) -> Co_output01\n", + "+kr='Co_output01': (4,) (dtype=int64 min=1 max=1)\n", + "Onnx-Identity(Sl_Slicecst2) -> Cu_CumSumcst\n", + "+kr='Cu_CumSumcst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-CumSum(Co_output01, Cu_CumSumcst) -> Cu_y0\n", + "+kr='Cu_y0': (4,) (dtype=int64 min=1 max=4)\n", + "Onnx-Add(Cu_y0, Ad_Addcst) -> Ad_C01\n", + "+kr='Ad_C01': (4,) (dtype=int64 min=0 max=3)\n", + "Onnx-Cast(Ad_C01) -> Ca_output0\n", + "+kr='Ca_output0': (4,) (dtype=float32 min=0.0 max=3.0)\n", + "Onnx-Reshape(Ca_output0, Re_Reshapecst) -> Re_reshaped0\n", + "+kr='Re_reshaped0': (4, 1) (dtype=float32 min=0.0 max=3.0)\n", + "Onnx-Mul(Ca_output0, Mu_Mulcst) -> Mu_C01\n", + "+kr='Mu_C01': (4,) (dtype=float32 min=-18.84955596923828 max=-0.0)\n", + "Onnx-Mul(Re_reshaped0, Mu_C01) -> Mu_C0\n", + "+kr='Mu_C0': (4, 4) (dtype=float32 min=-56.548667907714844 max=-0.0)\n", + "Onnx-Cast(Sq_squeezed01) -> Ca_output01\n", + "+kr='Ca_output01': () (dtype=float32 min=4.0 max=4.0)\n", + "Onnx-Div(Mu_C0, Ca_output01) -> Di_C0\n", + "+kr='Di_C0': (4, 4) (dtype=float32 min=-14.137166976928711 max=-0.0)\n", + "Onnx-Identity(Sl_Slicecst2) -> Un_Unsqueezecst\n", + "+kr='Un_Unsqueezecst': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Unsqueeze(Di_C0, Un_Unsqueezecst) -> Un_expanded0\n", + "+kr='Un_expanded0': (1, 4, 4) (dtype=float32 min=-14.137166976928711 max=-0.0)\n", + "Onnx-Cos(Un_expanded0) -> Co_output0\n", + "+kr='Co_output0': (1, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", + "Onnx-Sin(Un_expanded0) -> Si_output0\n", + "+kr='Si_output0': (1, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", + "Onnx-Concat(Co_output0, Si_output0) -> Co_concat_result0\n", + "+kr='Co_concat_result0': (2, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", + "Onnx-Transpose(x) -> Tr_transposed0\n", + "+kr='Tr_transposed0': (4, 3) (dtype=float32 min=-1.8227919340133667 max=2.354652166366577)\n", + "Onnx-MatMul(Co_concat_result0, Tr_transposed0) -> Ma_Y0\n", + "+kr='Ma_Y0': (2, 4, 3) (dtype=float32 min=-2.7854056358337402 max=3.9340782165527344)\n", + "Onnx-Pow(Ma_Y0, Po_Powcst) -> Po_Z0\n", + "+kr='Po_Z0': (2, 4, 3) (dtype=float32 min=0.0 max=15.476971626281738)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst3\n", + "+kr='Sl_Slicecst3': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst4\n", + "+kr='Sl_Slicecst4': (1,) (dtype=int64 min=1 max=1)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst5\n", + "+kr='Sl_Slicecst5': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Slice(Po_Z0, Sl_Slicecst3, Sl_Slicecst4, Sl_Slicecst5) -> Sl_output0\n", + "+kr='Sl_output0': (1, 4, 3) (dtype=float32 min=0.7225952744483948 max=15.476971626281738)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst1\n", + "+kr='Sq_Squeezecst1': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Squeeze(Sl_output0, Sq_Squeezecst1) -> Sq_squeezed0\n", + "+kr='Sq_squeezed0': (4, 3) (dtype=float32 min=0.7225952744483948 max=15.476971626281738)\n", + "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst6\n", + "+kr='Sl_Slicecst6': (1,) (dtype=int64 min=1 max=1)\n", + "Onnx-Identity(Sl_Slicecst1) -> Sl_Slicecst7\n", + "+kr='Sl_Slicecst7': (1,) (dtype=int64 min=2 max=2)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst8\n", + "+kr='Sl_Slicecst8': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Slice(Po_Z0, Sl_Slicecst6, Sl_Slicecst7, Sl_Slicecst8) -> Sl_output02\n", + "+kr='Sl_output02': (1, 4, 3) (dtype=float32 min=0.0 max=1.636295199394226)\n", + "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst2\n", + "+kr='Sq_Squeezecst2': (1,) (dtype=int64 min=0 max=0)\n", + "Onnx-Squeeze(Sl_output02, Sq_Squeezecst2) -> Sq_squeezed02\n", + "+kr='Sq_squeezed02': (4, 3) (dtype=float32 min=0.0 max=1.636295199394226)\n", + "Onnx-Add(Sq_squeezed0, Sq_squeezed02) -> Ad_C0\n", + "+kr='Ad_C0': (4, 3) (dtype=float32 min=0.8215643763542175 max=17.113265991210938)\n", + "Onnx-Sqrt(Ad_C0) -> Sq_Y0\n", + "+kr='Sq_Y0': (4, 3) (dtype=float32 min=0.9064018726348877 max=4.1368184089660645)\n", + "Onnx-Transpose(Sq_Y0) -> y\n", + "+kr='y': (3, 4) (dtype=float32 min=0.9064018726348877 max=4.1368184089660645)\n" + ] + }, + { + "data": { + "text/plain": [ + "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", + " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", + " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + " custom_fft_abs(x, verbose=1, fLOG=print)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "22.6 \u00b5s \u00b1 6.06 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs_py(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "269 \u00b5s \u00b1 4.46 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Again the gap is less on bigger matrices. It cannot be faster with the default runtime as it is also using *numpy*. That's another story with *onnxruntime* (see below)." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.68 ms \u00b1 55.9 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "bigx = numpy.random.randn(10000, x.shape[1]).astype(numpy.float32)\n", + "%timeit custom_fft_abs_py(bigx)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3.35 ms \u00b1 76.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs(bigx)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using onnxruntime\n", + "\n", + "The python runtime is using numpy but is usually quite slow as the runtime needs to go through the graph structure.\n", + "*onnxruntime* is faster." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", + " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", + " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "@onnxnumpy_np(runtime='onnxruntime')\n", + "def custom_fft_abs_ort(x: NDArray[Any, numpy.float32],\n", + " ) -> NDArray[Any, numpy.float32]:\n", + " \"onnx fft + abs\"\n", + " return _custom_fft_abs(x)\n", + "\n", + "\n", + "custom_fft_abs(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "108 \u00b5s \u00b1 46.8 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs_ort(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*onnxruntime* is faster than numpy in this case." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "248 \u00b5s \u00b1 6.11 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "%timeit custom_fft_abs_ort(bigx)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inside a FunctionTransformer\n", + "\n", + "The conversion to ONNX fails if the python function is used." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FunctionTransformer is not supported unless the transform function is of type wrapped with onnxnumpy.\n" + ] + } + ], + "source": [ + "from mlprodict.onnx_conv import to_onnx\n", + "\n", + "tr = FunctionTransformer(custom_fft_abs_py)\n", + "tr.fit(x)\n", + "\n", + "try:\n", + " onnx_model = to_onnx(tr, x)\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now with the onnx version but before, the converter for FunctionTransformer needs to be overwritten to handle this functionality not available in [sklearn-onnx](https://github.com/onnx/sklearn-onnx). These version are automatically called in function [to_onnx](http://www.xavierdupre.fr/app/mlprodict/helpsphinx/mlprodict/onnx_conv/convert.html#mlprodict.onnx_conv.convert.to_onnx) from *mlprodict*." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "tr = FunctionTransformer(custom_fft_abs)\n", + "tr.fit(x)\n", + "\n", + "onnx_model = to_onnx(tr, x)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", + " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", + " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from mlprodict.onnxrt import OnnxInference\n", + "\n", + "oinf = OnnxInference(onnx_model)\n", + "y_onx = oinf.run({'X': x})\n", + "y_onx['variable']" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/_doc/sphinxdoc/source/api/npy.rst b/_doc/sphinxdoc/source/api/npy.rst index 34d080983..e3d6ef02f 100644 --- a/_doc/sphinxdoc/source/api/npy.rst +++ b/_doc/sphinxdoc/source/api/npy.rst @@ -84,6 +84,8 @@ Decorators .. autosignature:: mlprodict.npy.onnx_sklearn_wrapper.onnxsklearn_classifier +.. autosignature:: mlprodict.npy.onnx_sklearn_wrapper.onnxsklearn_cluster + .. autosignature:: mlprodict.npy.onnx_sklearn_wrapper.onnxsklearn_regressor .. autosignature:: mlprodict.npy.onnx_sklearn_wrapper.onnxsklearn_transformer diff --git a/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst b/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst index 3ac18fdc4..5f2b25103 100644 --- a/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst +++ b/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst @@ -1,8 +1,8 @@ .. _l-numpy-api-for-onnx: -Create ONNX graphs with an API similar to numpy -=============================================== +Numpy to ONNX: Create ONNX graphs with an API similar to numpy +============================================================== Many people came accross the task of converting a pipeline including a custom preprocessing embedded into a diff --git a/_unittests/ut_tools/test_zoo.py b/_unittests/ut_tools/test_zoo.py index 8268a46a1..caa4ddc95 100644 --- a/_unittests/ut_tools/test_zoo.py +++ b/_unittests/ut_tools/test_zoo.py @@ -18,7 +18,11 @@ def test_download_model_data_fail(self): self.assertRaise(lambda: download_model_data("hhh"), ValueError) def test_download_model_data(self): - link, data = download_model_data("mobilenet", cache=".") + try: + link, data = download_model_data("mobilenet", cache=".") + except ConnectionError as e: + warnings.warn("Unable to continue this test due to %r." % e) + return self.assertEndsWith("mobilenetv2-7.onnx", link) self.assertEqual(len(data), 3) for k, data in data.items(): @@ -61,13 +65,21 @@ def test_verify_side_by_side(self): "Mismatch\n%s" % pprint.pformat(keep)) def test_verify_model_mobilenet(self): - link, data = download_model_data("mobilenet", cache=".") + try: + link, data = download_model_data("mobilenet", cache=".") + except ConnectionError as e: + warnings.warn("Unable to continue this test due to %r." % e) + return for rt in ['onnxruntime', 'onnxruntime1', 'python']: with self.subTest(runtime=rt): verify_model(link, data, runtime=rt) def test_verify_model_squeezenet(self): - link, data = download_model_data("squeezenet", cache=".") + try: + link, data = download_model_data("squeezenet", cache=".") + except ConnectionError as e: + warnings.warn("Unable to continue this test due to %r." % e) + return for rt in ['onnxruntime', 'onnxruntime1', 'python']: with self.subTest(runtime=rt): try: From 16d690dec7c15bb21b64e87af1a71e55e36697b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Fri, 19 Mar 2021 01:33:55 +0100 Subject: [PATCH 02/10] Supports embedded models in numpy API for ONNX --- .gitignore | 1 + _doc/notebooks/numpy_api_onnx.ipynb | 565 --------- _doc/notebooks/numpy_api_onnx_ccl.ipynb | 622 +++++++++ ...i_onnx2.ipynb => numpy_api_onnx_ftr.ipynb} | 102 +- _doc/notebooks/onnx_profile_ort.ipynb | 1124 +++++++++++++++++ .../source/tutorial/numpy_api_onnx.rst | 3 +- .../test_run_notebooks_onnx_numpy.py | 2 +- ....py => test_run_notebooks_onnx_profile.py} | 4 +- .../ut_npy/test_custom_embedded_models.py | 93 ++ _unittests/ut_npy/test_numpy_onnx_pyrt_skl.py | 47 + _unittests/ut_npy/test_onnx_variable.py | 26 + _unittests/ut_npy/test_onnx_variable_ort.py | 26 + mlprodict/npy/numpy_onnx_impl_skl.py | 13 + mlprodict/npy/numpy_onnx_pyrt_skl.py | 19 + mlprodict/npy/onnx_numpy_wrapper.py | 2 +- mlprodict/npy/onnx_variable.py | 20 +- mlprodict/onnxrt/ops_whole/session.py | 22 +- mlprodict/tools/onnx_manipulations.py | 49 +- 18 files changed, 2101 insertions(+), 639 deletions(-) delete mode 100644 _doc/notebooks/numpy_api_onnx.ipynb create mode 100644 _doc/notebooks/numpy_api_onnx_ccl.ipynb rename _doc/notebooks/{numpy_api_onnx2.ipynb => numpy_api_onnx_ftr.ipynb} (79%) create mode 100644 _doc/notebooks/onnx_profile_ort.ipynb rename _unittests/ut_documentation/{test_run_notebooks_onnx_memory.py => test_run_notebooks_onnx_profile.py} (91%) create mode 100644 _unittests/ut_npy/test_custom_embedded_models.py create mode 100644 _unittests/ut_npy/test_numpy_onnx_pyrt_skl.py create mode 100644 mlprodict/npy/numpy_onnx_impl_skl.py create mode 100644 mlprodict/npy/numpy_onnx_pyrt_skl.py diff --git a/.gitignore b/.gitignore index 35f560299..95dc417b0 100644 --- a/.gitignore +++ b/.gitignore @@ -312,3 +312,4 @@ _unittests/ut_tools/*.tar _unittests/ut_tools/**/*.npz _unittests/ut_tools/**/*.pb _unittests/ut_onnxrt/onnxruntime_profile*.json +_doc/notebooks/onnxruntime_profile*.json diff --git a/_doc/notebooks/numpy_api_onnx.ipynb b/_doc/notebooks/numpy_api_onnx.ipynb deleted file mode 100644 index af7509b3f..000000000 --- a/_doc/notebooks/numpy_api_onnx.ipynb +++ /dev/null @@ -1,565 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Introduction to a numpy API for ONNX: CustomClassifier\n", - "\n", - "This notebook shows how to write python classifier using similar functions as numpy offers and get a class which can be inserted into a pipeline and still be converted into ONNX." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
run previous cell, wait for 2 seconds
\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from jyquickhelper import add_notebook_menu\n", - "add_notebook_menu()" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext mlprodict" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## A custom binary classifier\n", - "\n", - "Let's imagine a classifier not that simple about simple but not that complex about predictions. It does the following:\n", - "* compute the barycenters of both classes,\n", - "* determine an hyperplan containing the two barycenters of the clusters,\n", - "* train a logistic regression on both sides.\n", - "\n", - "Some data first..." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAm7klEQVR4nO3dfZAc5Z0f8O9Pq92dud2V4GD8EgS7cKLupLgoBCvjFLgSGUF8LseAzgb2rghYy4GrYmLIlQ9fhLk/xFWdC4wtjHNrcsshX7wLhXEcx3EsW4YKduXIaoV0CUj2nQISXoI9gwxCEnrZ1f7yx7Mt9fZ0z3T39Ht/P1VT0s7Oy9M9s8+vn7ffI6oKIiIqnyVpF4CIiNLBAEBEVFIMAEREJcUAQERUUgwAREQltTTtAgRx7rnn6tDQUNrFICLKlZ07d76pqjXn/bkKAENDQ5ienk67GEREuSIiB9zuZxcQEVFJMQAQEZUUAwARUUnlagyAiKgTs7OzmJmZwfHjx9MuSiwqlQpWrFiB7u5uX49nACCi0piZmcHAwACGhoYgImkXJ1KqioMHD2JmZgYXXnihr+ewC4gobxoNYMcO8y8Fcvz4cZxzzjmFq/wBQERwzjnnBGrdMAAQ5cnkJDA4CFxzjfl3cjLtEuVOESt/S9BjYwAgyotGAxgdBY4dAw4dMv+OjrIlQKExABDlxf79QE/P4vu6u839VGivvvoqrrjiCqxcuRI33XQTTp48GcnrMgAQ5cXQEOD8w5+dNfdTod1777245557sG/fPpx99tkYHx+P5HUZAIjyolYDxseBahVYtsz8Oz5u7qfYRDnmfv/99+OrX/3q6Z83bdqELVu2tHyOquLZZ5/FJz/5SQDArbfeiu9+97udFwacBkqULyMjwPr1pttnaIiVf8wmJ80wS0+PaXyNj5uPIKyNGzdiw4YNuPvuuzE/P48nn3wSzz77LC699FLXx09MTOA973kPzjrrLCxdaqrrFStW4PXXXw9fCBsGAKK8qdVY8SfAPuZ+7Ji5b3TUxN+wp39oaAjnnHMOdu3ahV//+tdYs2YNBgcHsXv3bs/nvPnmm+HezAcGACIiF9aYu1X5A2fG3DuJv7fffjueeOIJ/OpXv8LGjRtx+PBhfPjDH3Z97MTEBFatWoW3334bc3NzWLp0KWZmZnDeeeeFL4ANAwARkYu4xtxvuOEG3H///ZidncXExAS6urpatgAAYN26dfj2t7+Nm2++GVu3bsV1113XWSEWcBCYiMhFXGPuPT09WLduHW688UZ0dXX5es6XvvQlPPzww1i5ciUOHjyI0dHRzgqxgC0AIiIPcYy5z8/P44UXXsDTTz/t+zkXXXQRpqamOn9zB7YAiIhaqNWAtWujqfz37NmDlStX4uqrr8bFF1/c+Qt2iC0AIqKErF69Gq+88kraxTiNLQAiopJiACAiKikGACKikmIAICIqKQYAIqKMe/TRR7Fy5UqISKSpIRgAiIgy7sorr8T27dsxODgY6esyABARtRJhPugw6aABYM2aNRiKYd8HrgMgIvIScT7oMOmgV69eHfr92mEAICJyE0M+6DDpoOOUWgAQkfMBfBPAewEogMdUtX1biIgoCTHlgw6aDrqoLYA5AH+iqi+KyACAnSLyY1Xdk2KZiIiMmPJBh0kHHZfUBoFV9Q1VfXHh/4cB7AUQzS4HRESdiikfdJh00I888ghWrFiBmZkZXHLJJbj99ts7KoNFVDWSF+qoECJDAJ4H8AFVfcfxuzsA3AEAF1xwweUHDhxIvoBEVAh79+7FqlWrgj2p0Yg0H/T8/Dwuu+wyPP3007FkBHU7RhHZqarDzsemPg1URPoBPAPgbmflDwCq+piqDqvqcI37oBLlU4RTKRMXYT5opoO2EZFumMr/W6r6nTTLQkQxiXgqZZ4xHfQCEREA4wD2qurDaZWDiGJkn0p56JD5d3Q01ZZAFrq94xL02NLsAroSwC0APiIiuxduH0uxPFQmee6SyBNrKqWdNZUyBZVKBQcPHixkEFBVHDx4EJVKxfdzUusCUtWfAZC03p9KjF0SyYlpKmVY1kyaRkEDf6VSwYoVK3w/PhOzgPwaHh7W6enptItBedZoAIODixf3VKvAgQPRbPpKzayA291tKn8G3MR5zQJiKggql5hWd1ILIyMmfUKEUykpGgwAVC4Z65IojVqNFX8Gpb4OgChRMa3uJMojtgCofNglQQSAAYDKil0SROwCIiIqKwYAIqKSYgAgIiopBgCioJhGggqCAYAoiMlJs5L4mmvMv5OTybwvgw7FgAGAyK+0MlumFXSo8BgAiPxKI7NlBtMpU3EwABD5lUYaiYylU6ZiYQAg8iuNNBLMXUQxYgAgCmJkxKSO3r7d/Bt3WmPmLqIYMRUEUVBJp5Fg7iKKCQNAWTUarFDyhLmLKAbsAiojTiukPOJaiMgxAJSN27TCT38a2Ls37ZLlAyuhdPCiJRYMAGXjNq3wxAlgzRr+UbXjVgkxIMSPayFiwwBQJo0G8NZbpsJ3OnGCf1StuFVCt97Kq9IkcC1EbBgAysK6er3xRmB+Hujqan4M/6i8uVVCs7O8Kk0C10LEhgGgDJxXrydPmsrerULjH5W7oSHg3XdbP4YBNB5cCxEbBoAycLt67ekBvvhF/lEFodr6950GUI4nePNYgMdT1hkGgDLwakLfeWeyq1rzbNcuYG6u+f7e3mgCaEyzXApVQdZqwNq1p88xJwZ1jgGgDFo1oR1/VBTQN7/ZeQCNaZZLkStITgyKBlcClwXTCXRmzRrTxz87e+a+7m5g3brOz6XVRXfs2OLX3r8/9GvbK0jrZUdHzVegCB99DKeslNgCKBNe7YdXqwFbtwKVCtDXZ/7dujWacxnDLBe3YZ/3dTXQ+EEx+oM4MSgaDACUL2l2ao+MAK+9Bjz3nPk3qjGTGGa5OCvImzGJl44M4vfuKkZ/ECcGRUO03cyGDBkeHtbp6em0i0FJsxLXvfgicM895tL25EnzF1+kgeuIE/RNTppun/d1NfDSkUH8Fmz9JdWqGbfIeY3JnIb+iMhOVR1uup8BoOSy/hdk1WJLlwKHDy/+XUEqsTg1GkDjBzvwe3ddgyWHD535xbJlZvB67dr0CkeJ8QoA7AIqs6xPE7GPZDorf4ALr3yo1YDVHxvCkjl2mFMzBoCyimoeXZx98m4jmXasxPxhhzl5YAAoqygSbMXdgnCb6gEAAwOsxIJKeitLygWOAZRVo2Eq7WMhBwY7fb5f1hiANQf/K18BLrss+JhF1sc6iGLEMQBarNNugaRS9DqvXO+8M/hahqyPdRClhC2Asgt7ZZxUC6BTeSknUYwy2QIQkcdFpC4iL6VZjlILuzo4LwOL3EyEyFPauYCeAPAogG+mXA4KIw/5hZgzgMhTqi0AVX0ewG/SLAN1KO78Qp1OMw3bUilUHuVs4ilOX+YHgUXkDhGZFpHpBr8p5eIcvH3ggXC1RdApkBw0jh1PcTakPggsIkMAvq+qH2j3WA4Cl4jb4C1gsnA+/nh889izNmhcwOmrWTvFZZDJQWAiT16rgI8fj3fnjywNGtsvky+4IHwLKCOsLp9du5pP8bFjwDe+kU65yowBgLLJaxUw4F4hR9WhnJVBY2eqjuPHzR7OOe0vsWLZ1VcDn/gEcPRo82NyHt9yKe1poJMA/g7A74rIjIiMplkeyhD74K2Ts0KOskM5K9NbvVpACex9GPXgrDOn34kT7tsrnzjBVkDiVDU3t8svv1ypZOp11c2bVSsV1WXLVKtV1YmJxb+vVlWBM7dq1dzf6ftOTXX+Op28v/O4rNuyZaZsMZiYMG+7fHnzqQ5rasp8fG6H4rxF8dFRMwDT6lKnsguIsq1WA+67z+zA5TaLx6vPfteuzqePprl9ptUSqVSafxdTl1RcG63395seLCe3Bg7X6CWLAYDywatCHhoC3n138X1HjgDXX99Zl1AWJqlbW1Bu3pxIl1Rc499HjrjHMZHm+7lGL1kMAJR/Iot/np/v7DLWa0whjaBgtYASSOUc1/j30FDzRwQAW7aYGb1pD7eUWdqpIIg6s3+/qTm8ZgwBZy5j/aa5tvpBrInqo6PAO++E2484qnn8tVrsNaPV62TPvu130XSrQ7S/bleXed0tW0xiVyD72USKLPWFYEFwIViHCrioyHPBmF2QVUY7dpgr/0O2/XP7+02tdeJEsNe09jKwBY3G+pHMfwReXxO3+10O0TMuFvHrlxdeC8FSn9kT5FbqWUCdzkqJY3pHFthnCQ0MuE8tGRsL9nrO2Te9vc2vPTCg+sQT3p+Hy+vM9lT1/Eo9lx+B29cnrglYFD1wFlBONRpmhcwFF4Qf1Ixjekcc/eFBX9Pqq3/oIdPJ/Ed/ZLaLtOvvNzuI+eW2DmDLluaJ64cPA3fd5f15uIyovntyKd57fH+kM2yS4PX1cVvRy1k8OeMWFbJ6K10LwLrs6nSy9NSUuXSLai55HK2JoK/pcvk5X6k2TzgPe0nqbHFZ5XNrZbi9h1v5AL0bDyYxnT869bq+/MSUXjRQb/r6bNvGFkBewKMFkHqlHuRWqgAQ5UKgKNvqcbT7w7ymS1B7G8t096c2m+e6LRrrVL1uun2cQcDr8xgba/rs5gG9HWMKmFi1bZvLYaa9CM2yEPRODSzXo6jqTZho+nisuBjH6aboMADkjdtVeycVblR/qVG3JsK+Zr2u846gcRSmj72xJ+IK1F4hBwlWU1OqfX1Nn98x9Or7l9a1p8elwZOVsRqX4zyKql7YX3ddjJ2FeEXeGADypF53b19bl41hK4Uo/lKz0gJQ1X2bJ/Qoqvo2lp2+Qr2w33RZRFYbuVXIfoNpvW4GkB2f4clqv17VO9V0uI09GRpVdQnKpwaW6ctPTPkrju27xgCRPgaAvLBXON3dqj09ZyqazZuz8VcUR7s/xGs29tT14z3bdD226bmo680wAeHUwPJoytUqMPmt1Vy6geZ6q6596i8/EUPrKqxOAr3tOzzbU9VbuidSb9CUHQNAHrj90Xl2FKcsjsu6IK+5UMmcqJr+6bt6x/QoIr56jqq7a2zMtAT6+1WrVT00NuH6MT/7VHO3VmSD2MF+bYQJ9B5dR+eizkHiFDEA5EEc/etF5FLJnOrp1VN9PgdnO3ifqCpke91qNfSWL1e9pXtCZ3s6bF21GUcINMwQNNB7DM4PY4pf5xQxAORBmVbWdNKC2LateXB1YKC5vz2KcxfjNBevoZ7zK3X9zbaQ58Zt+mm1qi9uq5/uufI1Uzbs58MWQCZ5BQAuBMuSLGxGkkTCsyAbuDjLMzkJXHdd85ZSc3NmwVbU5y7ohvIB1GrA2Wc3L6Y61FPDvrNDpqJ2WYD2zrFu/LsN+zE4aBZwOVMzNy3e6mSDHcd3eK6nis90j+Pkslr8X+csZHDNG7eokNVb4VsAlrSmTSQxBTFIK8dZnrEx74VxVllzNuUk8kZfmytwt1ulYnu/qAqU9CygrEyfzSiwC4haSqr7ye84h9+cPH19ph8lA8JWdJH3Mk2YcQT79Fivyh8wk8tO63T6ZxrK1HUaklcAYBcQGXHtBuLkN+m8V3mcz52fB9asibaMIXTSa+LZyxSyS6OxfgQXLTmA9diOQRzAU/DuturtPZOWGYDr53P88Cz+1V1D2d2PPqnvbgExACQtq/2Uce0G4mT1Edv/YOfmTO3XrjynTgXr50/oXEeRa69pw7MOIsr+/cA7vTVMYy3eROsO9y1bHKfP1oc/P7AM76KKjRjHK4dr2U1gl9R3t4jcmgXWDcAyAL/jcv8lrZ4X1y33XUBZ76dMKrGL36koXuXx09eS4LmOfPauV5fGnj2++pi8es8efNDc399vfm6VJbuxp67/9f4pHeprXrCWyWmcTErUEoKOAQC4EcD/A7AbwMsA1tp+96LX8+K85ToA5KWfMokRuyA1ZpjyJHyuI387t/NTrZpa22dAc9aHz4yZ89jYU/cdO/0mPs2MnE0ASFKYALAbwPsX/v9BAD8HcMPCz7u8nhfnLdcBgIu8zoi7gk7hXEd6AdoqE2yA82XVh798cELnetunyLDWJTgbZ9YyC15Y55dXAGg1BtClqm8sdBNNAVgH4D4R+bcANLI+qLJgP+UZca93SOFcR7pcwHl+envN/+18DnL+9DsN/PbnR9F14hiWHPYeoLCGHDZsaF4n0N8PfO1rse5HT2lxiwomYOB/wtH/D2AAwE8AnPB6Xpy3XLcAVNlP6RSgyR64dR/FuU67S8F6/z17AreYJibMlfwwpvQtLF/03FMDi1tD7Rocme72IV8QogvoEgArXe7vBnC/1/PivOU+AKimX6kkJcLjDD2e20kZsjZgHyCg2Sv0c1FvSpI317u4RvfaeqKvr3226zJ8lYsgTAB4BcCfwnQFWfe9F8B/8nqxuG+FCABlEGHlmcrYeVYH7H3WuM4K/SYs3jfh0Njiz8PrcFsloc1afKTWvOrsVmMAlwO4CMBuEfmIiHwOwBSAv4MZFCZqFvEG9Kms8cnqwiL7YoEWaxycQyBPYQSDOICP927HD8cOYNmdizvyvYZkrr3WfVgm4o+YUuQZAFT1LVX9DIC/BrAdwOcBXKmqX1fV+aQKSDkTceWZyth51gfs2ywSc1bolQrwuc01fOeXa7HhTveB9iCD2FmNjxScZwAQkbNE5BsAPg3gowC+DeC/i8hHkioc5VDElWcqCVKzkJXVi8/Lb3uF/tprwH33tS9+02pkD1mPj+Rfqy6gFwH8I4BhVf2Rqt4N4BYAD4hIFjOCUII8eyBiqDxjzMiciTcNlLGixeW383X8VuhBZTk+UkBuAwNmzAArWvzuj71+F+eNg8DZ4GsAkFNEfLHOZbsZN6d5bPjy5S/UtVJpPygb5cfCjzg/4DEILOZ3+TA8PKzT09NpF6PUGg3T7Xzs2Jn7qlVzkZzGFWCjYS6Kh4bydwXaaAArVizuTunpAWZmzP/tx7XoOLdPmm6f7m7MHZ/FRh3H384ubqG4fSaTC0/r6THvOT6+uGGT53NJrYnITlUddt7PbKAUSBQDgFEl6ewkBXNYUSYY3bWruS/95Engy19efFx33eU4Tpjuqbee3o6LlhxoqvyB5s+k3dBBGueSMsCtWZDVG7uA0tfpFPmo5o+nMVU/VNlb9JNs27a4/PbMnX7SAE1NuSdsczsXrdIjZXXZA0UH3BCGotDJAGCU88eTnooYquxtLqvXrDFltuvqaj4uy7loYBg78L6uBvbvB158ETh8uPlxlUrzZ9Jq5g6ndZYXAwAFFnaCTJQVTdJTEZ1lPxcNXLFkB3713F73PiEfEaNWA7ZuNRV2X5/59+tfN/vjON2MSRzAIH6Ma/DSkUG8/39M4p57mh/X1QX8/fYGRlbuaHovr8DNaZ0l5tYsyOqNXUD5FnVXQ5K59exlv3khtcJRVHXeOghnAQKkpLb3EtXrZo9e+3F9YdQ9n8/gbzVv9H5rj0n97NVP5dUj5XYuOcunOJDFTeFhFpj9AsA+AF9o93gGgOwJWklEXWnHWkk5XnxiQvX8SnNl7BrNQkQ7+xhDpWICQb2uqlNTJpe/I6PnVb1Ti17eLfFbkAhrP1zm+imWzAUAAF0A/i9MvqEeAH8PYHWr5zAAZEvYSiLTV5ZW4cbGXA/uN9umdLZvuXsAcF7hh8zg2VR3e/zymbG69vScuetDXVN6orq8dZl8ngIOCheLVwBYmkKvk+WDAPap6isAICJPArgOwJ4Uy0R2LSaG27u4+441cDH2496NQ1i/vuYr5UBa88xbznW3JsovXXpmdNVa8DA6Clx6Kc7GW8CpE+4v7uw4HxkB1q8H9u/Hm/1DePVIDUMN92O3xhjs6yus8ZHa2oUO/IW5/5idBcbHsWGkhpkNZjopAFx2/hB6Lu+8M79lWbg+oFjcokISNwCfBPDXtp9vAfCoy+PuADANYPqCCy6IK0CSU5vLe6uL2+oPfwvL9Siqum9z8L6CpFoELQ/Jz64o1p68PT2q3d1n9k50GwPw+74t3r7pqtvPiYqgj40tgOJBBruAfAUA+41dQAnxUQPU6+794fOVNonkHZLqa257SF67orQKCNu2md26WlTKQSrTUHW3W1CIIKJy87pi8QoAaXYBvQ7gfNvPKxbuow5EspzfRx9ArQb8x037MfvFHgBnHifHj5mNZefnm3MNuJTV6kay97SsX+8ou3VQ/f3AkSOhDq7tIbnNhQTM3ExVYMmS5ieffTawalXL9921yzzVzqs7xdZj5HmIrikhnLkdOuxjazSAlSuBnTtDn27KC7eokMQNwFKYXccuxJlB4H/a6jlsAbQW2dW038vWel3nO9hM1tdMSeugrPdp091yuvyOK2BfhzQ21nwMvb2qP/tZqD4Ra1/egKel7VTN5ctN62u2J/p+Gs7+KSZkrQvIlAkfA/APMLOBNrV7PAOAt8j7bf32AdjTWbabFRO0zK365dv1o7jUYK6HZK9t3XIrWMcQsE/Eq+jtnupVfOfrDWNK33Zs9h5mxk+7MrPvvxgyGQCC3hgAvPm6mg7aN+z38fW66Q93Xu76vEr2rFdb9cu7VXY+xy5OH5KztrWmfno9P8D5cyt6X585Ta1Oo9fbO1+v0zn/fsvcYUyhjGAAKLi2dV8SbfuQI4ee9WrQFkCQGszrhFlBoMNZNNu2Bb+aDpqw7ZbuCdMFF9FILVsAxcUAUAKe9W+Qv+xOZ5BEPafTOig/Uy6DHGe72tZ2DEEOyR5nu7vNjFG/9XO7xWA/3Dyl51fq3l1YEeDsn2JiACgJ1/rA75VxVkcArYNqM+VSVf3XYD6DhWd6Bp8v2dOj+tRTHabKsBVivmrWWsR5VZ7pldoUCgNAmfmp7IrU/vdbg7UJFnv2uOfm94orXkMWvb3BYumi4hfpc6HUeAUApoMuAz9J/IuUFN7vbugt8lpPTpp8/Sdcsj547QXgtZTgxIlg+x4sKn6RPhcKL8qt6GwYAMqiXRL/OJPCx/TljYQzWDQaeOtHO3DvxoZr5W9xq4OtONvb6+/xvjBZP8W4XycDQJm0ujLuZKuvVvK02exCWQc2XIOfHx/ETfAuq1cdPDJiVv86g0DoOjuuzyUPsnzhkJQot9Fz49YvlNUbxwASEOUIYJ76r13KehRVPRdm05WuLjMI7Hd2TOSzaco2MpvVCQlJi2hxBjKYC4iyKMpczXnKK+xS1ll0Y3V1P6a0hk2bgD/4A/+5cfzk9QkkzRzaSfOdJCrn/CTuirkLkF1AFJ889V+7lLUbs/jH2SGcOgU89BBw+eXAvn3+6yC/Y9HkUIaBb79dozF3ATIAUHxy0H99upsZpqxareIQluFdVLER43hjrobZWffuV2cXNbusI5KnC4cwgvbrt5vA0QEGAIpXjF/eTjVdhGEEu797ANf3bccgDuApNJfVuhB1Pveuu/Iz1p15Obhw6EiYFk5MzUkx4wP5MDw8rNPT02kXgwqg0TAVtX14orcX+MlPTCVuv9+uWjV58i+/3Psx1uMOHChOnZWKSDa3yCC3L1/MXxgR2amqw8772QKgyOSpC8TtIuzECeDqq4E//EMTDPr7zWO6uxdfiB450vxcJ68Lujydo9QVdRAlQy0czgKiSEx6bE7Vjq+LvAh2BHNqtWp3fNy81ews8MgjZoMzexkbDffn2rl1WYc9R1RAkU8TC8ltbmhWb1wHkE1hp/v7murdZkewTqbHT0y45/rxcxzOef6f/Wzref95WhJBxQMmg6O4hFmr4qtCbLMfwDNj9Y7XCnklfPNzHM7g0yoYZWKzFXsBg2z2U6YFaAXlFQA4BkAdCzNrz9dECLcHLZhf2o2vfG5/xyvkV60C/uZvTDdsf3/z71sdh7OLulWXddIzG5vGGuzTls47D1ixov2UpTyl8aBw3KJCVm9sAWRX0NQHnbYA5nqretFAPbIrautCN4INwTwltdmKs2vtmbEWLSkrX/WePYtfhH1WhQJ2AVHcgvYW+KoQFx50qrei84DOV8wYwKGxidjqpyD7z4R97bjqUbd6+6reKT01sNw7ALhtWpCJPiuKSrkDAPsxM8vPR/PMWF2v6p3S4b49elXvlLmi1c6uqNu979iYqRMHBvKVi8yt3r6wv65zvS1aAG4RlC2AQilvAGBWwVxrVw+Fie3tvhJjY63rxizzOl+HxmzRsrvb3NqNeHOD4MLwCgDFXgmcwoo7itaOHWYM8tChM/ctW2YyS6xd6/6cVmsLvL4SO3eaJQb9/e47gQ0MmFXCXu+ZJdZ6g+5uM9B8er2B/cS8+Wbzgbr9bRR1NW7JeK0ELvZCsDylIyZXQWfPtFts5faVAExdWKkAx48DIs2ve/JkfnKRea4xsqeUrtXM9CdnpHD+XZQpDXUJsQVAiQl7Mel5Revy+u0+brfH+DE2Btx5p/fvc3uhnNuCUxDlzAWUoZwbZdfJlHK/CUX9rC1wfiV6e83/7ZY62sWjo60r/1xPly9qvh3ypdgtAAuvclKVVEMsyPvY0wt1ktmTjUzKg3K2ACy8yklVuyvzqDJkBmnwWV+JVavatwhapWovw+ZVVFzlCAB+MVdvLFoN5EbdfRJm/xn7c3btav59q0HnjlM88DtHKWIAsOS6IzfbvK7MgWA74wV5v6EhcxXu97W8WgTtho06Gmbid45SVo4xgHbYkZsI51BMmDn+fkSRdz/osFHgYSZ+5yhB5VwH4BfXCyTCOaU8jgyZ9v22rY9zdNTMiw/yUQad/h54ujy/c5QB7AICks/VSwA6n6Xr1n2em0FZfucoAxgAAK4XSFGYQVugufv8gQdMIMhNvcrvHGUAxwDsuF4gF7xW81YqwOOPm//7WTmcCfzOUQI4BuAH857kglc+n+PHTcV/4IC55aJe5XeOUsQAQLnj1s1jWbLEzOW/9lrWq0TtpDIGICKfEpGXRWReRJqaJZRRGVm0ZO8+dzp6FLj+ek6pJ/IjrUHglwBsAPB8Su9PQWVg0ZI9/liDx5s3m/QNdlEtKCMqulQCgKruVdVfpPHeLWXkCjdz7JPro1yyG4Bb/KnVgPvuA773PaCvb/HjMzn1kyhjMj8NVETuEJFpEZluxFnhZOAKN22e8S/lyfXt4s+aNcD8/OLnZHLqJ1HGxBYARGS7iLzkcrsuyOuo6mOqOqyqw7W4RvUycIWbtpbxL+XJ9e3iD6fUE4UT2ywgVV0f12tHruTL8tumT7Bq2HbbB8bET/zx3AaRiDxxGiiQ+hVu2nzFP5caNqk1TH7jD6fUEwWT1jTQG0RkBsA/A/DfRGRbGuU4reR9CL7jn21jnUiHTHwMvodNGUFE3pgKwq7Ey/L9brwORJzJOIrczUTUklcqCAYAOs1v/AuTx9/1tZkTnygR5d4TmHzxu3Vy0CETz+6i3ORuJiomBgAKLMiQScsZtiUffCdKGwMAheJ3ULblRX7JB9+J0sZpoAkq2hizn2mXbS/yOYGfKDVsASSkrJkmfF3k+x18IKJIcRZQArIy2SXNFoj9vQFe8BMlibOAUpSFyS5pt0Csi/zt28OVg4laiaLHAJCAsJNdoqr0spLrLmw50g5eREXFAJCAMJNdoqz0stACCVuOrAQvoiJiAEhIkFw2UVd6WZluH6YcWQleREXEAJAgv5Ndoq70sjLdPkw5kgxeHGegsmEAyKA4Kr2sZNMMWo6kghfHGaiMOA00o4Jk5yyDOKewZmWaLlFcvKaBciVwRnGB7GJxbvZS8g3hqMQYADKMO1wlIyuD5ERJ4xgAlV5WBsmJksYWABHY5UblxABAroqWudQPdrlR2bALiJpwSiRROTAA0CJFSb3ARV1E7TEA0CJFSL1gtWDWrWMLhqgVBgBaJO9TIhsN4LbbTMvl6FHz7223sSVA5IYBgBbJ+5TIXbuaA9jJk+Z+IlqMs4CoCadEEpUDAwC5yuuUyDVrzuRPsnR3m/uJaDF2AVGmBZ3NU6sBW7cClQrQ12f+3bo1n8GMKG4MAJRZYdcjjIwAr70GPPec+bfMWVSJWmE6aMokpmgmio5XOmi2ACiTirAegSjrGAAok/K+HoEoDxgAKJPyvh6BKA84DZQyi+sRiOLFAECZltf1CER5wC4gIqKSYgAgIiopBgAiopJiACAiKikGACKikspVKggRaQA44PKrcwG8mXBxksJjyyceWz4V9dgGVbVpPl2uAoAXEZl2y3NRBDy2fOKx5VORj80Nu4CIiEqKAYCIqKSKEgAeS7sAMeKx5ROPLZ+KfGxNCjEGQEREwRWlBUBERAExABARlVRhAoCIbBaR/y0iu0XkRyLyT9IuU1RE5EER+fnC8f1nETkr7TJFRUQ+JSIvi8i8iOR++p2IfFREfiEi+0TkC2mXJ0oi8riI1EXkpbTLEiUROV9EnhORPQvfxc+lXaakFCYAAHhQVS9R1UsBfB/A/SmXJ0o/BvABVb0EwD8A+LOUyxOllwBsAPB82gXplIh0Afg6gN8HsBrAiIisTrdUkXoCwEfTLkQM5gD8iaquBvAhAP+mYJ+bp8IEAFV9x/ZjH4DCjG6r6o9UdW7hxxcArEizPFFS1b2q+ou0yxGRDwLYp6qvqOpJAE8CuC7lMkVGVZ8H8Ju0yxE1VX1DVV9c+P9hAHsBnJduqZJRqA1hROQvAPxrAIcArEu5OHHZCOCptAtBrs4D8EvbzzMArkipLBSCiAwBWAPgf6VclETkKgCIyHYA73P51SZV/S+qugnAJhH5MwCfBfDniRawA+2ObeExm2Caq99Ksmyd8nNsRGkTkX4AzwC429GjUFi5CgCqut7nQ78F4AfIUQBod2wichuAjwO4WnO2eCPA55Z3rwM43/bzioX7KONEpBum8v+Wqn4n7fIkpTBjACJyse3H6wD8PK2yRE1EPgrgTwF8QlXfTbs85GkHgItF5EIR6QFwM4DvpVwmakNEBMA4gL2q+nDa5UlSYVYCi8gzAH4XwDxMyujPqGohrr5EZB+AXgAHF+56QVU/k2KRIiMiNwD4GoAagLcB7FbVf5lqoTogIh8D8FUAXQAeV9W/SLdE0RGRSQD/AiZl8q8B/LmqjqdaqAiIyFUAfgrg/8DUHwDw71X1B+mVKhmFCQBERBRMYbqAiIgoGAYAIqKSYgAgIiopBgAiopJiACAiKikGAKIAFjJHvioiv73w89kLPw+JyA9F5G0R+X7a5STygwGAKABV/SWAvwLwlwt3/SWAx1R1P4AHAdySUtGIAmMAIAruKwA+JCJ3A7gKwEMAoKo/AXA4xXIRBZKrXEBEWaCqsyLyeQA/BHCtqs6mXSaiMNgCIArn9wG8AeADaReEKCwGAKKARORSANfA7B51j4i8P90SEYXDAEAUwELmyL+CyRn/GszA70PploooHAYAomD+GMBrqvrjhZ//A4BVIvLPReSnAJ4GcLWIzIhIbrOaUjkwGygRUUmxBUBEVFIMAEREJcUAQERUUgwAREQlxQBARFRSDABERCXFAEBEVFL/H+VteQo7i0p6AAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from sklearn.datasets import make_classification\n", - "from pandas import DataFrame\n", - "\n", - "X, y = make_classification(200, n_classes=2, n_features=2, n_informative=2,\n", - " n_redundant=0, n_clusters_per_class=2, hypercube=False)\n", - "\n", - "df = DataFrame(X)\n", - "df.columns = ['X1', 'X2']\n", - "df['y'] = y\n", - "ax = df[df.y == 0].plot.scatter(x=\"X1\", y=\"X2\", color=\"blue\", label=\"y=0\")\n", - "df[df.y == 1].plot.scatter(x=\"X1\", y=\"X2\", color=\"red\", label=\"y=1\", ax=ax);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Split into train and test as usual." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.model_selection import train_test_split\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The model..." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1,\n", - " 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1,\n", - " 1, 0, 1, 0, 0, 1], dtype=int64)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy\n", - "from sklearn.base import ClassifierMixin, BaseEstimator\n", - "from sklearn.linear_model import LogisticRegression\n", - "\n", - "class TwoLogisticRegression(ClassifierMixin, BaseEstimator):\n", - " \n", - " def __init__(self):\n", - " ClassifierMixin.__init__(self)\n", - " BaseEstimator.__init__(self)\n", - " \n", - " def fit(self, X, y, sample_weights=None):\n", - " if sample_weights is not None:\n", - " raise NotImplementedError(\"weighted sample not implemented in this example.\")\n", - " \n", - " # Barycenters\n", - " self.weights_ = numpy.array([(y==0).sum(), (y==1).sum()])\n", - " p1 = X[y==0].sum(axis=0) / self.weights_[0]\n", - " p2 = X[y==1].sum(axis=0) / self.weights_[1]\n", - " self.centers_ = numpy.vstack([p1, p2])\n", - " \n", - " # A vector orthogonal\n", - " v = p2 - p1\n", - " v /= numpy.linalg.norm(v)\n", - " x = numpy.random.randn(X.shape[1])\n", - " x -= x.dot(v) * v\n", - " x /= numpy.linalg.norm(x)\n", - " self.hyperplan_ = x.reshape((-1, 1))\n", - " \n", - " # sign\n", - " sign = ((X - p1) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel()\n", - " \n", - " # Trains models\n", - " self.lr0_ = LogisticRegression().fit(X[sign == 0], y[sign == 0])\n", - " self.lr1_ = LogisticRegression().fit(X[sign == 1], y[sign == 1])\n", - "\n", - " return self\n", - " \n", - " def predict_proba(self, X):\n", - " sign = self.predict_side(X).reshape((-1, 1))\n", - " prob0 = self.lr0_.predict_proba(X)\n", - " prob1 = self.lr1_.predict_proba(X)\n", - " prob = prob1 * sign - prob0 * (sign - 1)\n", - " return prob\n", - " \n", - " def predict(self, X):\n", - " prob = self.predict_proba(X)\n", - " return prob.argmax(axis=1)\n", - "\n", - " def predict_side(self, X):\n", - " return ((X - self.centers_[0]) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel()\n", - " \n", - " \n", - "model = TwoLogisticRegression()\n", - "model.fit(X_train, y_train)\n", - "model.predict(X_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's compare the model a single logistic regression. It shouuld be better. The same logistic regression applied on both sides is equivalent a single logistic regression and both half logistic regression is better on its side." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0.82, 0.84)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from sklearn.metrics import accuracy_score\n", - "lr = LogisticRegression().fit(X_train, y_train)\n", - "accuracy_score(y_test, lr.predict(X_test)), accuracy_score(y_test, model.predict(X_test))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, this is true in average but not necessarily true for one particular datasets. But that's not the point of this notebook." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[-0.01589338, -0.11623031],\n", - " [-0.37916406, 0.41219093]])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.centers_" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[-0.82405569],\n", - " [-0.5665088 ]])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.hyperplan_" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([[-0.63926519, 0.37923185]]), array([[-0.90877675, 1.7896483 ]]))" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.lr0_.coef_, model.lr1_.coef_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's draw the model predictions. Colored zones indicates the predicted class, green line indicates the hyperplan splitting the features into two. A different logistic regression is applied on each side." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":20: MatplotlibDeprecationWarning: shading='flat' when X and Y have the same dimensions as C is deprecated since 3.3. Either specify the corners of the quadrilaterals with X and Y, or pass shading='auto', 'nearest' or 'gouraud', or set rcParams['pcolor.shading']. This will become an error two minor releases later.\n", - " return ax.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABF00lEQVR4nO29eZgcdbX//zq9Tc8MJJlMJoSsEzYTiZAQyBBkiSwS81VQBA244DUY4Iq4kavITzEgyFVEhSiQS1BABa9BIF4JGJBVIJBAhIRE2YZshEwmk2223j6/P7p70t1T3dN71XSf1/PwMFNdXfWpTk+dOud9FjHGoCiKoijpcNm9AEVRFMXZqKFQFEVRMqKGQlEURcmIGgpFURQlI2ooFEVRlIx47F5AKRgxYoRpbm62exmKoiiDhtWrV+8wxjRZvVaRhqK5uZlVq1bZvQxFUZRBg4i8m+41DT0piqIoGVFDoSiKomREDYWiKIqSkYrUKBRFqQyCwSCbN2+mp6fH7qVUDH6/n7Fjx+L1erN+jxoKRVEcy+bNmznwwANpbm5GROxezqDHGEN7ezubN29m4sSJWb/PttCTiIwTkSdE5HURWSciX7fYZ5aI7BaRNbH/fmDHWhVFsYeenh4aGxvVSBQJEaGxsTFnD81OjyIEfNsY87KIHAisFpEVxpjXU/Z7xhjzcRvWpyjOp60NWluhuRmaLFPgBz1qJIpLPp+nbR6FMeY9Y8zLsZ/3AuuBMXatR1EGHffeCxMmwBlnRP9/7712r0ipUByR9SQizcA0YKXFyzNF5J8islxEjsxwjPkiskpEVrW1tZVqqYriDNraYN486O6G3buj/583L7pdKSqtra1MmTLF7mUAcMABB9hyXtvFbBE5ALgf+IYxZk/Kyy8DE4wx+0RkDvAgcLjVcYwxi4HFAIc11pplF0wu3aIVxWaGtXczM9RLYt5KMNTL858/jl2Ntbatq9iMuXQRu94Ol/Qc4VFHZHy9oztAOGJo7wqUdB0AoVAIjyfzbTnTOhrrfMVeEmCzRyEiXqJG4vfGmD+nvm6M2WOM2Rf7+WHAKyIjyrxMx7PB383S4e1s8HfbvRSlTHTVe5FI8nRKiRi66rNPeaxUXmxbw02v3cGLbWuKdsxwJMw3vnopHz52Kud+Yg4bXn+dj5zQ0vf6W2++0ff7tMlH8MOrruSk447hjJM/zNtvvQnAjrY2vnTBZzn9pBM4/aQTWPn8cwD893XXcum8/2DOabO4dN5/cO89d/P5z3yas2afwXFHfZCfXP+jfuvZt28fn5pzJh85oYWTjjuGh/9vGRD1fiZPnsxXvvIVjjzySD760Y/S3V34fcHOrCcBlgDrjTE3pdlnVGw/RGQG0fW2l2+VzmeDv5vvj9vI70e08f1xG9VYVAkBv4c1LaMIuYWg10XILaxpGUXAb3uQwFZebFvDJ1d8hevX3MInV3xlQGMxkDcR5+0332Te/Ev4x6o1DB02jFf/uYYhQ4fy2j//CcC999zNBV/4Yt/+Q4YO5ZmXXuaiiy/lqv+6AoDvLfg2l1x2OY898xy//cN9fOOrl/Tt/68N67n//5bzP3fdA8DLq1bx29/fx9MrV7Psz/fzysurk9bj9/u5+74/8cRzK3lw+d+4+srvEB9r/cYbb/DVr36VdevWMWzYMO6///6srjETdn6rPgx8AXhNRNbEtn0PGA9gjLkNOBe4VERCQDcw1+iQ7yTW1nUREkNEIIRhbV0Xk3oqJ/SgpGfrhKHsOKieus4gXfXeqjcSAM9uW0UgHCBMhEAkyLPbVjGjaWrBxx3f3MyHjj4agKOnHsOmje/y+Qv/g3vvuYsPTvkpD97/J/721D/69j/nvM9E//+Zz/L/fXcBAE8/8Xf+vWF93z579+xl3759AMz+fx+ntnb/3+2sU09jeGMjAP/v7E+y8rl/MO2Y6X2vG2P40Q+/z/PPPovL5eK9rVvZ/v771Llg4sSJTJ0avebp06fT2tpa8PXb9s0yxjwLZMzTMsYsAhaVZ0WDkylddXiMEMLgMcKUrjq7l6SUkYDfowYigRNHHYvP7SMQCeJzeTlx1LFp983WmwCo8dX0/exyuwj1hPjEJz/FT3/8I06cNYujpx3Td2OH5BTU+M8RE+GRJ5/B7/f3O35dXfLfbWoKa+rvS++7lx07dvD4P17A6/UybfIR9Pb2UFfro6Zm/1rdbvfgDj0pxWFSTy3XbhrP53Y0ce2m8UzqqVXNQqlaZjRN5cEz/ofvHX0ZD57xP2m9iVyMRDr8fj+nnn4GC75+Oed//otJrz24dCkADyz9E8fOiGoXs049nf+59Vd9+8TDVlY8+ffH6di5k+7ubpb/ZRkzZp6Q9PqePbtpamrC6/XyzFNPsmlj2g7hRUEfRSqAST21feGmuGYRkqiHETceilItzGiaWpRwUzac+9nz+euyZXzk9DOStu/a1cHJM6bjq6lh8W/vBuDHN97Ef33r65w8YzqhcIiZHz6Rn938K6vDcsyxx/Klz81l65bNnDf3gqSwU/y8nzvvHE467himHjOdwz/wgdJcYAypxJD/YY215qYzm+1ehi0sHd7O70e0ERFwGfjcjibO3dk48BsVxYGMuXQRh445qKjHLIY3EWfRL25i7549XPmDH/Ztmzb5CB575jkaR+SXoHnvPXez5pXV/PdNv8z5vdmmx65fv57Jk5NLCERktTHGMlanHkWFoZqFopSHL849j9a33+aBhx+1eyklRz2KCmSDv5u1dV1M6arTsJMyqCm2R1FMb8KJqEehZE2qZqFGQ6l2Kt1AlBo1FBWMCtuKohQDTY+tYJKK8SRajKco1YZ6E4WjhqKCiQvbLoMK24qi5I2GniqYeDFeokahmoVSTdjlTbzb+g5fufALdOxs56hpx3DrHb/B5ytNZ9dyoB5FhTOpp5Zzdzb2GQltIKhUC3aGnK75/lVcctnlvPTaeoYNG8bv7vqNbWspBmooqgjVLJRqYEe7m5dfrWVHEWY4/fjahdy26Oa+36/74Q+4/Ve3ZHyPMYZnnnqSsz51DgBzP/cFlv9lWeGLsRENPVURWoynVDpLlw3h8ivH4vEJoSD88tYw55wXyft4n/vihVx4/me55LLLiUQiPLD0f3ng4UeZdfxxlvvf/pu7GdHUxNChQ/sGEI0eM4b3tm7New1OQA1FFWGlWShKpbCj3c3lV46lu8cFPdFtX7/UzcmzIoxoyu+Y4yc0M3z4cF5ds4a27e/zoaOnMm78BJ584aW072nfsSO/kzkYNRRVhhbjKZXKxi0+vF5Dd8/+bR4vbNwojGjKvwPF57/0Ze773d28v/19LvjChezdu5dPnHGq5b63/+Zujpg0id27d/eNNd26ZQsHjx6d9/mdgBqKKkWL8ZRKY/yYAMFQsuwaCsL48YW1Kfp/Z53NDT9aSCgYZPFv7sbtdmf0KABOPPkUlj3wZ8457zPc9/t7+NjHP1HQGuzGzlGo40TkCRF5XUTWicjXLfYREblZRN4UkVdF5Bg71lqJqLCtVBLhUUfQcOSh/PLWMLW1hgOHGGprDb+8NZx32CmOz+fjxJNP4exzzsXtdmf1nh9cex233vJLjvvQZDp27uRzF/5HYYuwGTs9ihDwbWPMyyJyILBaRFYYY15P2OdjwOGx/1qAW2P/VwpEhW2lEjnnvAgnz4qwcaMwfrwp2EgARCIRVr/4Ikt+94es39M88RBWPP2PgXccJNg5CvU94L3Yz3tFZD0wBkg0FGcDd8fmZL8gIsNE5ODYe5UC0GI8pVJIrZcY0URBmkQi/1q/ngvO/SRzPnE2hx52eFGOORhxhEYhIs3ANGBlyktjgE0Jv2+ObetnKERkPjAfoKnOEZfleHQynmInvp4QdZ1Buuq9jp37/YHJk1m97l92L8N2bP/XEZEDgPuBbxhj9uR7HGPMYmAxROdRFGl5VUOSZkFUs1BDoZSK0a27mfriNoxLkIhhTcsotk4YmvNxtOFfebC1MltEvESNxO+NMX+22GULMC7h97GxbUqR0QaCSrnw9YSY+uI2PGGDNxjBEzZMXbkNX08op+OokSgftnkUIiLAEmC9MeamNLstAy4TkfuIiti7VZ8oDVqMp5SLus4gxiUQ3u/4G5dQ1xl0bAiq2rHzX+XDwBeA10RkTWzb94DxAMaY24CHgTnAm0AXMLhzzBxOomYBlV2QNxji45VKV70XiSRHhyVi6Kr3Zn0M9SbKi51ZT88CMsA+BvhqeVakJFLJ4nax4uNKfgT8Hta0jGLqyuR/g0oy2Hfc9mtu/9UtvPP22/zr3S00jhhh95IKonL+ZZSiUqnidmJ8PB76mLpyGzsOqq+oG5XT2TphKDsOqs/LqxsM3sSM40/gox+bw9mzP2r3UoqC/mUollRqQZ7Gx51DwO8piYGQtjZcG98lMn4CpqmwirsfX7uQhoYGLrnsciDaZnxEUxMXf/VrGd931NSpBZ3XaehfhmJJqrgNsHR4+6DXK4oRH1eci+9//8gB/3kxxutFgkH23Xo7gfM+m/fx8mkz/oHJk/M+n1NRQ6GkJS5uV5JeUQ3x8UokG29C2to44D8vRrq7ke7o9MYDLr2Yjlmn5u1Z5NNmvBLRvw5lQCpNrygkPq44F9fGd6OeRPf+Eb/G68W18V3CBYSgcm0zrh6FUpVUol6Ra3xcsY9stYnI+AlIMJi0TYJBIuMnFHT+fNqMVxo6M1sZkLhe8bkdTYM67KQMPsyQkdnv29TEvltvx9TWEhkyBFNbG/29QEE7nzbji3+9iA8dfghbt2zm5JZj+fp/XlLQGuxGH6mUrKimYjxl8BI477N0zDq1aFlPkF+b8fn/eRnz//Oygs/tFNRQKDlTSeK24lw6F6wAOnJ+n2lqKkiTSETbjEdRQ6HkTKWJ2/mgLUCqA20zHkW/4UrOVKK4nQt2twCpBiMV9SaiGGOI9hBVikG0M1JuVOa3TCkp1Twdz+4WIHYbqXKQaCR6jJu9uzo4cFiDGosiYIyhvb0dv9+f0/vUUCh5Ua3T8exsAWK3kbKDrZEDoG0n/h1tdi9lULDdN/D3wO/3M3bs2JyOW5nfLqWsVJNmYWcLkErvU5XoScQJi4tNZgjozMqsOH9ybgYgW7SOQimYapqOF28BEnILQa+LkFvK1gJE+1QpdmHrY4iI3Al8HNhujJli8fos4CHgndimPxtjrinbApWsqLbpeHa1AKnkPlVW3oTiHOz+hv0WWATcnWGfZ4wxHy/PcpR8SdUsKt1o2NUCRPtUKXZg67fMGPO0iDTbuYZqo9SpldUkbNtFpfWpUm/C+QyGb9tMEfknsBW4whizzmonEZkPzAdoqhsMl1V+ypFaWU3CtlI4aiQGB04Xs18GJhhjjgZuAR5Mt6MxZrEx5lhjzLFDKuhpq1gkplZ6gxE8YcPUldvw9YSKep5qErYV+6npaGf4ujXUdLTbvZSKxtF3VGPMnoSfHxaRX4vICGPMDjvXNRixSq0EGNLRw46DDyjaeaqhGK8aKqPLQaHexPjlD9JyzQIiXi+uYJCVV9/IxtlnF2l1SiKO/paLyCjgfWOMEZEZRD0gfXTIA6vUSnfYMOPpzaw5/uCihqAquRjPKnyn4nL5qelop+WaBXh6e6C3B4CWhVfwfsuJ9DY02ry6ysPu9Nh7gVnACBHZDFwNeAGMMbcB5wKXikgI6AbmmnwalVQxiU+/a1pGMW3lNlxhgwACeCKlre6tJM3CqjJ62vPvgUuIVHBLjVJQqDdRv3UTEa+3z0gARDxe6rduUkNRAuzOejp/gNcXEU2fVfLA6un3xZPGcNwzW6I3uxilrO6tpAaCVuE7lwEJG9xV0lKjGBRDwO4cPQ5XyjQ7VyhI5+hxBR9b6Y/TxWwlT9KJ1111/at4S1ndmzodD2Dp8HY2+LsHeKfzsArfpRI3ukpp6W1oZOXVNxKq8ROoP5BQjZ+VV9+o3kSJ0MeeCiVdXyBvKFL26t64ZjHY9YqA38PaY5o4+qXtxPuYpvYzLZbRrUTBvNipsBtnn837LSdSv3UTnaPHJRmJPR0u2ra6aRodZkhDpKjnrUYq4xuo9CNTX6BdjbW2CLCVoFd01fv6bTNA2A1QnL5Ppax32d3TwPbOMYys38JQf+7T45xGb0NjPy/iueW1LL6mAbcXwkGYf3UHJ8wefB6sk1BDUaEM1BfIjureStAr0k1EeP3okWydMKTgz7SUrcSfap3Dohd/hMcVIhTx8LWWqzh5wvKCjpkt5Sqs29PhYvE1DQR6XdAb3bZ4YQNTWnrVsygANRQVjNP6AlVC88DdDX4iAu4EZy0iFMVIQOlaie/uaWDRiz8iEK4lEI5uu2XldRx90Asl9yzKWX3dttWN20ufkQBwe6Lb1VDkjxqKCsdpfYESayxg8DUQDPg9vDLzYKa+8B6IgDGsOf7gon3GpWolvr1zDB5XqM9IjKCNSaxjb8cIhh48+ENQcZpGhwmn5BKEQ9HtSv445w6iVB2DVdwupadWqlbiI+u3EIpEjzGXe1nCPIJhL/XP7GNNy0Elq/0ody+nIQ0R5l/dweKFDbg9USMx/+oO9SYKRA2FYhv5iNtOyQYqpadWCkM01N/B11qu4t4XvsGSyDzq6Aa6IVx5tR8nzO5mSkuvZj0Vkcr4Zig54ZSbbbbidny9Q3b2MOWV7SXtfusUSmGITp6wnFN9j+F+JggJkZhSFVza2Rl2SENEDUQRUUNRZZSj1Xi2ZNNAsG+9Ap5QtPVIsbOBqglXQzdukuP1pSi41PbhlYX+hVURxU69LIZnkqmB4A1vjWZOfL0WlLL1SKVS6nGqaiAqE/0LqyKKmXpZCs8kVbNY7+u0bI0ep5StRyoZp6VNK85Hez1VEcVKvSzVEKTUoUeTA/X91muAoEcIuYtTBV2tBPwedjXWFvXzU2+ictG/siqiWGGHUhWFpWoWh0Zq+6137TFN7GmozetJ2CkifiWiRqKy0b+WKqMYYYdSFYVBf81i6bQQ0yaOZnqbp6AbvJNEfEUZbGjoqQopNOwQ90xCbiHodZUkDBQXtn8/oo3vHrqVF8aQ9/HLNS+8WlFvovKxe8LdncDHge3GmCkWrwvwS2AO0AV8yRjzcnlXqVhRakG0mJ1mSxUqU5RqwW6P4rfA7Ayvfww4PPbffODWMqxJyZJSCKJxUoXtQjrNljJUVu2oN1Ed2D0K9WkRac6wy9nA3bE52S+IyDAROdgY8155VqjYRVzYXu/by3FtPg4xXl4dll8DwUJEfBXA01MKI6EDh5yJ07/5Y4BNCb9vjm3rZyhEZD5Rr4OmOqdflpINp24I8K0XOzAuYeXB27j6ixBwk1cDwXxCZSqAlxcdOORc7A49FQ1jzGJjzLHGmGOH6JPfoCdVgH5mPARd0dkP4ZhmkSu5hMqcKID7ekIMa+92hAhfbG8iceBQ9z4XgV4Xt/9wOFve0b9lJ+D0f4UtwLiE38fGtikVTqoAPasVfGEIGPBFYNqu/iNJS3l+sFcAT/JuwhH+fWQjGw9rqIhw2J4OF688W4PLQ9LAoWAArpw7kksWqmdhN07/li0DLhOR+4AWYLfqE9VBqgA9czM8fhc82QwnbhbMER52JYxKLraW4CQB3KpH1+TX2vnA6zt5pczhsGJ7Es8tr+X2hQ2ICwI9qYNmhVBQdJSpA7A7PfZeYBYwQkQ2A1cDXgBjzG3Aw0RTY98kmh77H/asVCk3cQF62sptuMLRrrEzN0f/C7nhlg+HeGVYO1O66jh1Q6DoWkKpm+flgpV3I4A7Fg4rdQfd3T0NbO8cw8j6LUW9YezpcPHrHzQQCSdGwOPXuN9oGKOjTO3G7qyn8wd43QBfLdNyFIcRF6DHv9nBEevaMW4XEjHc/dHhfPfQrYTE4DXCzMcNnjBFbz/ulOZ5Vt5NnFKHw55qncOiF3+ExxUi5Kpn/nHFCwM9fE89kXCqF9GfYEDw11lfv1IenB56UqqcgN/Dm1Oa2HhYQ98N+8nRu/uK8YLG8NRE4aSNyVrC0I4egj53wTd4J8wc7/NuXngPdyTxWbu04bDdPQ0sevFHBMK1fbO2ixUG2tPh4q+/O5Dkq4ni9hjCof3bfTWGnq6BDYpSOtRQKIOCxBt24mQ8L8Kst5OfNt3BCDOe2UKkwFCUk2oo4t7NhDc7OOL1nUnXVqq1be8cg8cV6jMSAG5PccJAbVvdeGuiM60TcbnB5SJltBI0jU7dopQTNRTKoCOxy+y0XT5mbtnC82OjQvesVjh+M7jCBncBoah0NRR2Go+A38MbU5p4N8G7KuUaRtZvIRRJPn44VJybdtPoMJF+Wb6GL323g7p6w+KFDbg90fPNv7pD9QmbUUOhDEriXWaHtXfz3HjhzM8ZAu5oCu1jd8EJm/fvm2scP90kQE8gkvfM7mIamHKFw4b6O/hay1XcsvrGot+0hzREmH91B4sXNuByQygIFy7YxWmfjtbHTGnp1QptB6GGQhnUdNV7WT0+aiTCrmidxZPNyYYi1zi+ZQ2FwIde3o47kvsY2cFc4T190be4uWOb5U07U7uNbFpxnDC7O61BGNIQUQPhINRQVDhOirOXiokyBF94T18x3lF7fBgCfa+/e8iQnK49XQ1FxCW4E+5d2Xgq6byTt4eNYVOomZH1Wxjq78h6beUkXjNhddPO1G4jl1YcahAGB5V55xjEJN7YgYJu8qV4knWS4Um8vvHvwP0z6jjENZTZ67Yl5dJMeHsPb0xpynq9VjUUa49pYsrLbUn7ScQQ9LgY1t6d9vOw8k5CxssvHrmJf7qnEop4+FrLVZw8YXlen4EdJLbbiFdSx7OhgLSvqUEYvKihcBCJNz5XOAIGIh5XXjf5dE+yhdQXlDKEkqsBSr2+EzfC8Vt6eOmk4RiX8PzBpk/cPvb93GsNrGooQl53kvF4d+IQTnm0NePnYeWd1EQCtLCS5yMnAXDLyus4+qAXHOVZpKvArulox/PsexzkcbOp96C+7fFsKAC3l6RWHMXKlFLsQw2FQ7C6sUM01RNyv8kXu1dRKQxPnHwMkNX1dYYP4NX3jyN08AOc+QX6xO1H74nkVWuQKhonGo+gx8Upj7YO+HkE/B7WThvJ0ave7/NyBLiJBezjQO7gYjyuEG93TOYA3x5Hh6LGL3+QlmsWEPZ4Ob8zxJdZwh+J1swmZkOFg8nvK1amlGIfFdM9drDTd+NLQ/wmny3F7lVktb5c12RFvl1ao9eXvM1LkGv+9QvumTl8v7jtgvtOOrAoYbJEr8cbimT9eewZ7ifkTt4mwC18nRG00Ruq4fpnfsUPnriTi5Y9ztPvfizpnOXsGNu5YIWlN1HT0U7LNQvw9PZQ07mXOrq5k3mMq3sfX02kLxsqns3kq4lQWx9Jek0ZvKhH4RAytWmA3G/yxe5VVKomefl6PgG/h0c+eCynvvYaQbx4CfJlluBxhTl410Q8pgMTMXhEmOAbDj0FLbOf17N22sisP4+uei8uI+zvYxS7BrwcJhtYJS0Ewn4CYT+wPxQ1eVtrWbOlMjX8q9+6iYjXC737P0h3nYerv/MKoROPSjIEmbKZlMGJGgoH8cYHh/dV3VppFLne5IvZq6hUTfIKMUBd4yKcu/aPBE0Na5jG6TzG+tCReB4PcPp6w30nHRg1EsDS4e05T8aLYxV2m/LK9j6Be6DPI+D38NoxyeEngFpXN+ccfyuvvjSVUGR/23SPK8TejhFMffGFkoT68qFz9DhcwWRvyR0O0nDiwfRaGAKrbKaajnbqt26ic/Q42mhSQzKIUEPhABKfVo0xvDFpOBsPawAKy3qC4hZnlaJJXr4GKPqZ/YtPuM7FhN38l9zAT8x3qaMbQnDSRph53z5umTukr4FgPpPxIL3Xs6ehlsfOOjSrz2Pj4Q0ghimrYwV7Bv7ZMpJRB7X2q34ORTw001rWeRgDtQ/vbWhk5dU30rLwCiIeL65QkJVX30hvQ2PG98WJ6xtRryTElyN38OfauTrJbpCghsJmrJ5Wj3h9Z99QGrtTUFMpxZpyNUCJn5knVi/xC/kmERdJTYKMS1jv6+xrIBiKTcbL1VBk8npy+Tw2HjacbWOHJF3nUGLVzyuvw+MKEQx7OO+Dt7Gvrqaoob6BssqyKZDbOPts3m85sc8ryNZIJOob8dDVYi7i0X1nsIMmTZ8dBDjrLlSFOG2Sml3kcsO1/swMLosb6+RAPR6zmxBRj2JKV11eaytW2M3qOk+esJyjD3qBR978DEtfv4QHNlzEn16/hMAhn2bu248UfM6BsspWTHmWxXOyK5DrbWjM2kDEsdI3gnhpppUdNGn67CDA7sFFs4FfAm7gDmPMDSmvfwn4KfvHny4yxtxR1kWWGCdNUisXhRbtBT0uXOGUz8zA2ukj+2kGh0YO7GsgmK9GAeWZTbH09YuTRO15b9+P78yTGRVqy/ucVh7r0S9s5+1hY/AP3cd7Fz3O7R9rIBjIXCCXqC/kaiis9A0vQVppBjR9djCQ8ZsnIkOAJmPMWynbjzLGvFrIiUXEDfwKOAPYDLwkIsuMMa+n7PpHY8xlhZzLyThlklq5Kq5zrZlIXVff+zEYIOyOysPx46SGdmB/A8E4G/zdeRmOUoYCrVp6e1whNoWa8Tfuy/u4Vt5XV6SeXzxyE1On/41//fpAgoHkNN/UJ/xEfcEVjGoTG2efnfUaUvUNAiHmR+6g09+IL1T69NlCjJwSJe23XkQ+A/wC2C4iXuBLxpiXYi//FjimwHPPAN40xrwdO999wNlAqqGoeOyepFaupnW5Fu31S0mNZRl5Er0JY3hq9kQ6h9YAA9/MN/i7+f64jQWJ26XAqqV3KOJhZP2WNO/IDiuP1UuQNyKTeP6lE+ElSB0eFAruf8K30hdaFl7B+y0n5nTTTdU3TqKJSVvbSp71VKiRU6JkKrj7HjDdGDOV6Kzqe0TkU7HXijFuagywKeH3zbFtqXxaRF4VkaUiMi7dwURkvoisEpFVe8pUnFRMAn4PuxprbfEk8il4y4dcivas1jVl9XZMyjfPuF14Q9nfaNbWde0XtyUqbheb3T0NvNE+hd09DVm/J97S2+fups67F5+7m6+1XFVwlXbA7+HeiR+ji1p2M4QuavkyS9hBE9E/49Q/ZcMnL9rTd/Pu0xcSCLu9dDz7Hns6cqvX7W1oZOeRU+ltaGRIQ4RDjwyW3JOIGznfvr14entoWXgFNR3tJTtnpZLpruQ2xrwHYIx5UUQ+Avxf7GZdrgG2fwHuNcb0isjFwF3AqVY7GmMWA4sBDmus1QG7WVJOMd3q6dYVjjbWy3ZdrpT7Sq56TuJ0vHzF7UwkzZjOseFfXNTe3jmmXyuPfEODu3sauOidpXyLfTTTSivNMSNhjbfG9M2EAGt9IdwVYuFPpvH+9U2OTm21EtEjHi/1WzdpCCpHMj0S7BWRQ+O/xIzGLKLhoSOLcO4tQKKHMJb9onX8nO3GmHh7sTuA6UU4r5JAOcX0uB4Tcgsht2AAg+GUR1sZ/e7ugddl4LVjRhJyC0Gvi5BbMuo5Vu0v4tPxPrejiWs3jQeixXgb/IXf7BJnTHcFDyQQruWWldfl7Fkc3rg2yUiMbt3N6cveYuYTmzh92Vv9PqtMxLWPHTSxiuNiRsLQ/1nP4K2JcHGKXhDXF0I1fnrrDuzzSDZ1HkSg18XihQ05exblwsrIuUJBOkenDUwoacj0aHIJKX6pMWZvLFPpyiKc+yXgcBGZSNRAzAUuSNxBRA6OezXAWcD6Ipy3rDipLbcV5RbTt04Yyu5hfk555B0E8EQATD+tIt26tk4YyrZxBw74mWbSXeLidrH1inSC9PbOMXmHkNLpOruH+fGGIgN+r6y0D68PzvvP3Sy9dQhuT1ST+ORFezjt012WoaCNs8/mzQ+czNZHt/Hr3x3F5q7+XWOdmNpaaJGgsp9Md4MHgdtE5GfGmDCAiBwE/AyYBFxTyImNMSERuQx4lGh67J3GmHUicg2wyhizDLhcRM4CQsBO4EuFnLPcDJbJZuUW072hCMbtgsj+m4tVuCvdugYSrLMVzZP0ijyL8RIphSBtFYIDOOWRdzDugVvQ940zXXkdbn9N3zjTj7Rs5tOHb6GVCdRPash4o48OIhqNy3MkPV3JmobTU1vzLRJUksl0R5gO/BhYIyJfBz4EfAv4CfDFYpzcGPMw8HDKth8k/HwlxfFeyk4p23KXgnJWgecS7spnXdnqLsXWKxJvyokaRSGCtNVn5Q6bqKsfya4FfVz7ePecB/DXGQ55/AHOWvitATOB9nS4aN3g5faFyXUWYPDXGSLh4s3QLiX5FAkqyaT9CzTGdACXxIzEY8BW4HhjzOZ071H2oxXX6Sl1uCtbQxTXKwotxkskkyCdD6mflSscrSDx5DiSteuypbx6fw3P3tHLm4Fv4yFzumt8nKm46Fdn4a8zXPidDqadqG03qoVMdRTDgP8GWoDZwBxguYh83Rjz9/Isb/BSjRXXuVDKcFcuhmigYrx8NKah/o6iDh+yGpiUKEYP9L16qnUOt3xsFMGAcCyrCOCLNk+MkZoJlDTq1IJIGDUSVUamb/7LwK+BrxpjQsDfRGQq8GsRedcYc345FjhYcUrFtZPJJay0u6chp6f0fAxRqrh964vD+eKKnY7QmBI/q1y+V7t7Glj08k+joSOglWZ8sUaKcVIzgdq2uvuNMwVDjd9gTPpwUzaNBZXBSaa/npNTw0zGmDXACSLylZKuqkKwu+LaLoqd6ZVvbUKu+kaquP1eVzueMI7TmHL5Xr17zgO4V9B3099BE19mCXcyjyBe6msC/TKBmkaH+40z9dYYvvmzHTRPClkagXioKpvGgsrgI5NGkVaLMMb8T2mWU3k4sVV4KSl2pldibUI87TQ+Aa7Ys6UTxW1vRDh5IySGeJykMSV+rzIZZqub/h+Zy9O+U/nmRa9y6KdH9hN64+NMFy9swO2hL1PqqJnJnkicpFBVhsaCyuDF/m+8UjGUItOrFLUJ6UgUt6ft8jFz01aeHwtPNsOsVjjuPedpTJkMc+eCFQwh+aa/v2YizJCGycnRpQRyGWdqFapycn2FkjtqKJSiUYpMr1I1y0tHorh990eHc9kx7QTc4AvDopeHM8IB3kScdIZ547XLk7yEfGdYW40ztcLKa3F6fYWSG86svVccS6aGd6XI9CpVs7xsePIQF70eCLug1xP93UlYNVmM+A+gfuumfvuWsglfPFTlq4lQWx/BV1P61uFKeXHO45HieAYSlUuV6VXs2oRsKXUDQStyye6ybLIYy2DKJgOpmFlK+XotyuBAjKm8RquHNdaam85stnsZFcXungYuWvY4gfD+mgOfu5s7zjqt3w3N6f2tciHfIUf58Mgb53HHy1fhcQUJG3dW2V2j39293zC7fKy8+kZ+s+987vrpMDw+iISsM5A0S6kyOX/a2LzfKyKrjTHHWr02uP+KlbKRi6icKdMr13qIgSj28VJJ1CziRqN510EMbZtU1HMuf+M8blu1EBCCkegQpltWXsfEYRvoCdUnnSvxmplAX6rsju88xF8fH8ed1zcAQiimG6RmIGWbpaR1EUocNRRKVhRDVC5kVkM5jpeJeDFeEDDD9lHzj59jNn24KOfc3dPAHS9/j9QhQiYifPORB/C6A33XZ4xYXnPH959gT4eLu3/a0O84LndyBlI2WUrqcSiJOEudUxxLoaJyMWY1lPJ4A7G2rougy4VxGXAF6B37Uk7ntJqNEWd75xi8rv7bg6aGYKQm4fqu55aV/a/5vYseB+Dhe+r7vIhEEkebwsBZSokeR/c+l+PnTiilRz0KJWsKEZWLXQ9RzvqKzgUrOHTnatz/mEsoBER80Dor63MOVIQ4sn4LYeNOeZfB5+olEPH3bRHCSMxZGEEbzbSyXUbQtrWOFx/3s+y3Q7AabXrhgl1JoaN0BXW5eBxKdaGGQsmJfBveFbseotT1FZ0LViT9fsTw6XzrqD/ys1+uJfzmR2DzTBj7PO6JD4L/Zep3G8tBQtkUISbNjJAwwYiHzx/1c/7w2jeS1mBwY4xhLveyhHkE8OELBXh89c+45tf95owBhs/PfZvzJq+ns2Nc1rUVWhehpKJZT0rZePrdj/Wb1VBIfL/Yx4P+BiKV5x6pZfHCBhj/POHPnAruAL4wPHYXzHgveqNO9BiGtXcz84lNeIP7b8RBr4vnPzKOXY3JWVSpwrxVFlR9sIvvvHR3UvfXoM/PIZ7WpMlzABfI77nLd9GAcycyXWeix6Hpr86nIrOeYmNVf0l0wt0dxpgbUl6vAe4mOkSpHfisMaa13OtU+pNPtlGx6yGKfTwrI1HT0Z40HS3+JL7s5T/x6N4AYRcEDDzVDCds7u8x5FKEmOitPdU6hyWvXInHFSQY8fCV6ddx8oTlDGvvxuMJRGc+xoh4vIwLtrKZ/YZiBNu50/0VPL2Z506kI9XjWPtCDZfPGaXidpVim6EQETfwK+AMYDPwkogsM8a8nrDbPKDDGHOYiMwlOh/js+VfrZJIIdlGxZ7VUOjxMhmHYevXMv1nC/s9kQ9piHD6sLE8sStqJHyRaC+oOIltS/IpQkwU6uMsefl7zBz7OL76EC63F0L7w0DucJBTFgxj9c8iuNwQDgmXf+5V5E8e2Lf/uKlzJwYi3sJDm/4pdnoUM4A3jTFvA4jIfcDZQKKhOBv4YeznpcAiERFTifEyB5GpYC61m+sI2njuhU8xY9gz+IfuS3NEZ7G7p4F3z3kgGkIh+UY3fvmDtFyzgIjHjbezMxr1T3kiB5gaOJhH/+Dl2TFBZrXCzM30NRA8cWMkyWNIbAu+zdPEplAzI3vSe0AZhfrGDlZefSMtC68g4vHiCkUN2FGz67n5tG19HkATTbj+kCw0pM6dyBYVtxU7DcUYILEpzWai0/Qs9zHGhERkN9AI7Eg9mIjMB+YDNNWpRp8vA2XoJN7E4qJqMOKl7pFO/nn8yJxbipe6YC6VFVOejdYHrOgfQqnpaKflmgWxcE3/90Y8Xg5d+juOvHMREa8Xdy+csNGDcbt5bmwvp18IATd4DFyzOciknv3fw4Dfw4ptZ2XliWUS6jsXrKATeL/lxKSQGCQ38eul0dKg5DM7WsVtpWLuqMaYxcBiiIrZNi9nUJJNhk78JjaCNpYwLyaqdkMk95bi5SyY61ywIhpCmZM+hFK/dRMRr7fPg0jFFQpw5JJb8AR6+/YJ1dTw9E1LuMv1JL3v30mECMZE6y4SW37kMlcjMQsq8bMZ6u+gM7ZPb0Njv5t+qp6ycfbZlgYlVwZKp1UqHzsNxRYg0Q8eG9tmtc9mEfEAQ4mK2koCxXoqz6ZNePwm9twLnyIY8QLJguaQjh52HHxAVmvO9sYZD4UFPS7LFNR0pOoPA4VQOkePwxVMfnSOfxIRr5d1877G5Ltvh8D+A0Q8PoJDhnLIwR/H0/Y7QuEeywaCudZ9WAn1qdeT2GJjygt/jobMUvQUK4OSK3s6XBw0LsR1f9hOT5do1lMVYqeheAk4XEQmEjUIc4ELUvZZBlwIPA+cC/xd9YlkivlUnm2GzskTljNj2DPUPdJJYojfHTbMeHoza44/eMAQVLY3zngoDGNwRyDs7p+Cmkq6FNeBQii9DbFwzQ+/jTvQi7C/MsGIi02nzeHIJYuS3h+P+x/R0Mj3TriP9e3Pc8wj9/VrIOj3dBIM+5K25VL3kXpNiS02GgJtvGP+C08wvwynTFi18jj0SIvyb6Wisa0m3xgTAi4DHgXWA/9rjFknIteIyFmx3ZYAjSLyJvAt4Lv2rNaZFLuNRTxDJ+QWgl4XIbekzdDxD93HP48fSdgtfU/dAnhiISirVhWJZFMwlxgK80Rixw8bPGFjeY7OBSvoXLCCmo52hq9bQ01HsvOZzdyEjbPP5pmfLyFUm+wRRLw+vF2drLz6RkI1fgL1BxKq8SfF/Y8YPp2zD7+McV97lg3+bpYOb2eDv5unWufwrUfvJ2pVDT5394AtUJ5qncNFyx7nB0/cyUXLHue5R/YbntQWG6MDG+kOJhvzeIZTIWgrDyWOrRqFMeZh4OGUbT9I+LkHOK/c6xoslKKNRWKGzkAhnq0ThhL0uTnumS1RXSNGNlPtMsXh41iFwlLP0fH9J5K292UtpSkys6pITo3td0yagkSSQytxz2HnkVMHjPv/e+dqrh+3kZAYPMZF5PFPEEpIdY0Y4Rezz2Hc0HcsPxursNzihTV9WkpqCK2VZrwUJ8MpEc12UuJUjJhdjWT7VJ7rbIhMbcJT2d3g77ct26l2AxXMWYXC+s7h8rHjOw8lbUvOWkofgknMDkpnWDJlDA0U91/f/jwhl4sIYYImgmfik7BxVt/rPneQnlB92vdbPQAk3qBTQ2g7aGK++w5+65lXcIZTIprtpMRRQzGIGeipfKBU12JQ6FS7TAVzicfu0yhqoobJ6kZolbWUqcgsk2EpJGNocuNMPC4voQi4XF7MO7OSXh9Im7B6AEi8QVtlITVcfSbLWl4oOMMpEc12UuKooRjkpHsqzybVNZFCptLlEq7Kla0ThrLx2uXUb91EsK4eb1dn2huhVdZSphDMQIYl1XPIdpDPEcOn8wl5gAeffBnXllMIb5mJe/zTuCb+HfPOLL42dlnG0OBQfwfzr+1i8cKafjfoeJjsIy3jmPJwcgitl8IznFLREacKqKGoCKyeyrNJdY1TDM8jl3BVtiRm+mRzA+zLWsqyyCwXwxLP/nF5IBSACxfs4rRzuyyPu6fDxV9+9FHCvbMJA4x9Hr4wm4i7B+8p1zJy0zgYYKyq1Q16IP2lVCSG6pTqRA1FhZJtqmuunkc5GKiDayZyCRllY1j2dLho3eDl9oUNBAP7C/WWXN+AETj90/2NRT8RuPlJcAcwLkMw0r8YL5X49SfeoLPVXxSlFKihqFCy1Q5y8TxKTSEGIpFciswyGZa4FyEuCAZSZz0I9/y0gRmn9vR72u4nArfOgrAPTAAT8dG86yCS2r9mQa76i1JdxEOSjK2BpqaiH18NRQWTjXaQSxvsfBhI+yiWcSiEVMNS09EOG7bw54XTCQTS1wy4PcYyVTQuAt92dQOhoESHHN31ODQ/Sc3m4xh6xO3QuNbymOk+j1z1F6V6SAxJEg7BkiVw/vlFPYdWzlQ4Ab+HXY21ab2DXIrscmV0625OX/YWM5/YxOnL3mL0u7v7XosXxzmN8csf5Kw5xzPnirm8EZjIZ7k34dVkgxoJp08VPWF2Nz++bzveeDH25pnw7JWY905ld9OGvmK8bImHydIV+1UT6Qoqq5HEkKRv317o7oZ586Ctrajn0Ql3ClBY1lO6452+7K2kQryQW1j2t1cce3Or6WjnrDnHR3WAGF3UMoF32UETLncEl0vweA2RcHbDe1InxX3i+3/jL5FP9fWEunbT+D69IhvDmVocWG3YJejbQTb/1sPXreEjl14QNRJxhgyBxx6D447L6XyOnXCnOIdiZy1ZaR8R/wGOjqlb6QBBvDTzDjvdjXz64j3MOK0np8Z4qdlLT+x4ktD6IBGBEKZP2M7WuypGk7/BSjUJ+tkaRKuQJMEgNDcXdT0aelJKQle9F3ElN8Fzakx9T4eLt9Z52VY3od8fnZcgrUwkEhb+ctcQrrpgJO9v8uSULjqkIcKhRwYZ0hDpK8Zz4cbj9vfrMqukp8+QJ1CMnlZOIzWc5OntoWXhFZahttSQJLW1UY2iyIK2ehRK0YkP11l53ENFGZxTSpK7ozYR+eTNfP6By+kK+PAS5MssYQfRP7qezmjmU+oY0MRCPCBjcdoRw6f3dZmd3DiTcR+fzis7V7P+jUVMbpzJEcOnl+nKBx/VIujnmuGWmLl35pktmvWkOBer0EmxBueUCqtZ0Jc8OI+u22by+4s7eSN4SJ+RSCSx71KioQn2CiZi8NX2n56XyBHDp/cZhH/vXM31z80lFAnicXn53gn3qbFIQ64FlYOVfAxiX0iyBEYC1FAoBTJQbN3JMXWr7qgisLl7JNMXulm9sAG/O0JPV+Jkiv19l6wMDQjdsdHhqZ6HFevbnycUCRIhTCgS/V0NRXqc/vBRDJxoENVQKHmRadraYGn3YNUdtbdH+Nm3RvDFb+/iWze1A4a2LR7u+dmwfn2X3lrn7WdoEsnUkjv+eU044IS+BoIel5fJjTOLfp2VhpMfPoqF0wyiLYZCRIYDfwSagVbgM8aYfl3SRCQMvBb7daMx5qzUfZTyYuVBWE1BGyhtFLI3LjUd7QzbsBYBOiZNKdofTbww7vYfNsQqr6P/BXuFJdc34K83RGKG4eaHt/Vbq5WhSSRdS+7kz+vjfOL7D+D5wJOqUShJOMkg2lJHISI/AXYaY24Qke8CDcaY71jst88YM/AA5hS0jqK4ZAov7elwcfmcUdHwSwxfTYSbH96W8eafrXEZv/xBjv/BN3GFoy0vIl4vLyz8eV+qYDE8mVefr+HnVzTS222dBJjpehLrJIKBmEbh3+95pF5TNp/Xv3eu7hO71XAouXD+tLF5v9eJdRRnA7NiP98FPAn0MxSKvWST25/PFDSr2L5VPL+mo52WhVfgDu/vi+QOBmlZ+G3ebzmRJ14Ym5cnk0rzpCAmg43JdD2pdRKQOetpoM+rnOJ2YkEX4Jgwh+I87DIUBxlj3ov9vA04KM1+fhFZRbSD2g3GmAfTHVBE5gPzAZrqVHophFxaa+QzBS1b41K/dRPG7e73fuNyw4YtLL7mQwMam2xIHNDjcpNWvM70/sRzZjr/QJ9XKcRtK68rsaDL3d0NLiFc48+q2rnaq8OrkZLdUUXkMWCUxUtXJf5ijDEiki7+NcEYs0VEDgH+LiKvGWPestrRGLMYWAzR0FMBS69K8u27lM8UtGyNS+focUi4/w1aImFamVDUec6JnsE7G7z87sb+4nUxGOjzSpyOVwxx2yrE95GWzf0qnAlHvTXIXO1cTS00lP3YpVH8C5hljHlPRA4GnjTGfGCA9/wW+D9jzNKBjq8aRfYUqzFfrlpBag+ktBrFIw9x/Pe/0U+jWNvyqby0kVyvx19ncmrZkevxrY6bqFEAeesV6fSQe296hI9/Z25yf6AEgrV1PHvjYrbNPCVpu1UvrFCNn2UPv6CehUOoNI1iGXAhcEPs/w+l7iAiDUCXMaZXREYAHwZ+UtZVVjDF7tya6xS0bEdsxtMEfS+9zu6dQqjlSHwTGxhC4fOcM92shzREWPtCTVE0ECsyfV7xgrxC9Yp0Ib5W+rcqScTT3cVJ35zHyh/+LMlb0JkY1YtdhuIG4H9FZB7wLvAZABE5FrjEGHMRMBm4XUQiRHtS3WCMeT2bg3tCEXw9IdsmtDkVp7X1zta4REXrD0Vv2L/cf8MuZJ7zQFlXezpc3H5NA8EiaCD5UqhekS7EVz+pIamgy93TDSK4QkHi6own0NsvBFUtLTSU/thyJzXGtAOnWWxfBVwU+/k54EP5HL9+b4DTl72V1+znSsRpBiIXBsqQsjI2A4XB0h1zwgeCfWGmx5fWEexNnmpXiAaSD4XqFZn0kNSCrmEb1nLSFfPxdu8f7ZrqLTixYlgpDxX5yC0GPGFj++xnJzCYjQTknn6bTX2G1TEBrpw7Em8NhAIQiSRnPgGEBsh+KjapDQTzyX7K5HUlFnTtmjQFiSR/nlbegtMqhpXyUNF3ULtmP9uFk41CvoVxuaTfZlufYXXMQG/UMIT6tqcmeRg+OW/PgGsvdiuTxAaCkF8xXjYhvly8BSdVDCvloaLvoMWc/exknGwgIP8WH5Bb+m223kfqMUNBATH9Qk2pHDjAzbaQ68yGUhfjqbegpKMiDYURCLmKN/vZqTjdQED2T/mZyFa0zsX7SDymv85w1QUjk153uQzRSEzceAi/u3EYM07tyUn3KKb4XY5Os+otKFZU5IS7zgN9PHbWoRUpZHcuWNH332Cg7yk/gfhTPuyfLrenI/NXMXFKXKZ95l/dga8mQm19BF9NJGPKbPyYYyaG+r3v05fswV+fHH5KXHeu11kMkqbjaadZpYxU5ON2yOOqOE9isBiGVDI95ZciVJNvyqxVz6aHlgyxXLcV+bQyyZVUcRvgIZ2Mp5SBivQossHXE2JYeze+ntDAO9vIYPIerEj3lA/0hWq697kI9LpYvLBhQM8i23M2jQ7TttWd0/ESvZZ8vJNc9reipqOd4evWWM5GjnPE8OmcffhlAFz/3Fz+tP5Grn9uLv/euTrr8yhKrlTWY3eWjG7dzdQXt2FcgkSMI+stBrNxSMXqKd9q6E+x6hSK5ank6p0UUgCYaw8lnYynlJOqMxS+nhBTX9yGJ2wgHI1BO6XeopKMQyqpKZqlCtUUW1TOtTVJrvtDrJ16SpO+TI35oPjNAxUlE1VnKOo6gxiX9BkJsL/eopINRDry6TqbDfnMx7CbfHooFaMYT1GypeoMRVe9F4kkZ7PYVW9RjQYikUJCNWBd3FYOUbnY5NtDqRjFeIqSDVVnKAJ+D2taRjF1ZbJGUS5votqNQyr5hGqgvw7x+St2MXFSkKbR4ZJ4KqWkGD2UyjkZT6k+qs5QAGydMJQdB9VT1xmkq95bFiOhBqJ4WOkQd17XgL/eEIkZhpsf3lbUVhqlptCqaBW3lVJSlYYCop6FGojBiXVTP6GnM1pFvXhhAzc/vI1Dj0w/c8GJFFIVreK2Ukqq1lCUEjUOpcVKh0jE6eJ1KbASt1WzUIqFGooiogaiPCRmTLnc0NOV3BI8GAB/XfWNTU8Ut1WzUIqJLZXZInKeiKwTkUhsql26/WaLyL9E5E0R+W4515gLg716OhuyqRouNYl9oU6Y3c3ND2/jqtvbmPe9aEW01xcBDC4XXHXBSJ57pNa2tdpNsmYRZH3783YvSRnE2OVRrAXOAW5Pt4OIuIFfAWcAm4GXRGRZtuNQB8LXEypYzK504xAn16rhUpCu2jredmPS9ADfO/8gQGKzJco/utRJqGahFBO7RqGuBxDJ2P9/BvCmMebt2L73AWcDBRuKQlp4VKJxyDRsJ5+q4VKsb6Bq654uweMzBAP7v1PVqFXE0YI8pZg4WaMYA2xK+H0z0JJuZxGZD8wHaKpLf1n5tvCoRAMBA/dFyqdquNhkU209GAvtSk2qZqFGQ8mXkhkKEXkMGGXx0lXGmIeKfT5jzGJgMcBhjbVplcxcW3hUqoGA7J7U860aLibZGIFStQSpBFTYVgqlZIbCGHN6gYfYAiTejcbGthVENi08Ktk4JJLNk3oxqoYLJVsjUGhLkEpFi/GUQnFy6Okl4HARmUjUQMwFLij0oJlaeFSLgYiTbbjGqmo4k65RCrI1Avm2BKlkVNhWCsUWQyEinwJuAZqAv4rIGmPMmSIyGrjDGDPHGBMSkcuARwE3cKcxZl0xzp/awqPj+08U47CDjlzCNYlVw8WcTFfT0Z512wo1AvmhxXhKoYgxlVeYdFhjrbnpzOaM+1Sb95CJXLyDPR0uLp8zKqprxPDVRLj54W0538SdkHZbjahmUbmcP21s3u8VkdXGGMu6NieHnkqCGoj+5PKkns+8BytD5IS022pFNQslV6rGUKiBKA65pqGmC1M5Ie22WlHNQsmVijcUaiCKSy66Rqb02xoHpN1WK1qMp+RKRRqKyEGH07ngYbuXUbFkm4GUMUx1pP1pt9WMFuMpuVCRhsIplDuFtJxko2sMFKYqdFiPUjgqbCvZYEv32GrgueW1XD5nFNdf2sTlc0ZVZSfTeJjKVxOhtj6CrybSL0zV29DIziOnqpGwCe0yq2SDehQlIJvWGOVci51eTWqYCuCtdd6K9LIGIypsK9mghqIE5JNCWgqKWRhXCPEwVSHrsdvgVSpajKdkgxqKElBIJ9Ni3RCd5NUUuh6nGLxKRSfjKQOhGkUJyCY2b0UxdY0+ryaBuFdjB/muJ9HAdO9zEeh1sXhhA3s69KtbClSzUKxQj6JE5NrJtNgegNPmM+S7HqeE8aoF1SwUK/SxrITEx3Rmc0MrtgeQr1dTKvJdT7kNXuJc7mokrlmcN/kKDTspfahH4RBKcUN02nyGfNZTzoFEqoVESdQsQAvyFDUUjqFUN0SntebOZz3lMHhOE/+dgorbCqihcBRO8wCcRKkNnmoh1minWQVs0ihE5DwRWSciERGx7H8e269VRF4TkTUisqqca7SLXHQNpXg4Tfx3CnFx24Vbxe0qxi7Fbi1wDvB0Fvt+xBgzNd1ADUUpBk4T/51CqrgN8NAbi/j3ztU2r0wpJ7aEnowx6wFExI7TK4olGvqzJi5uq15RvTg9B9AAfxOR1SIyP9OOIjJfRFaJyKq9HTvLtDyl0tDQX3q0GK96KZlHISKPAaMsXrrKGPNQloc50RizRURGAitEZIMxxjJcZYxZDCwGOOSDR1XeIPAyoT2VlHRoMV71UjJDYYw5vQjH2BL7/3YReQCYQXa6hpIHlVZHoEavuGgDwerFsemxIlIPuIwxe2M/fxS4xuZlVSyVVkdQaUbPKWgDwerErvTYT4nIZmAm8FcReTS2fbSIxGeYHgQ8KyL/BF4E/mqMecSO9VYDTmsiWAh7Olzcro0ES45qFtWDXVlPDwAPWGzfCsyJ/fw2cHSZl1a1VFIdweNL6wj2JmfUafFc8VHNonpwbOhJKS/l7KlUSvZ0uHhwyRAg2VCEBqnRczJWmoVSmaihUPqohDqCtq1uPD4IBhK3Gj45b8+gvB6now0EqwM1FEoSTmsimCtWITSvz3Dap7vsWVAVoeJ25aLqnuJY8pkNYdWK4+IfDr4Q2mBExe3KRT0KxZEUkt5aCSG0wYiK25WLGgrFcRSjpmOwh9AGIypuVy5iTOV1uxCRNuBdu9dRACOAHXYvosRkuMYD6uDwI8CVUMQRCcMb/4Z9g01sqPJ/y4qi0q9zgjGmyeqFijQUgx0RWVXpbdWr4RqhOq6zGq4Rquc6rVAxW1EURcmIGgpFURQlI2oonMliuxdQBqrhGqE6rrMarhGq5zr7oRqFoiiKkhH1KBRFUZSMqKFQFEVRMqKGwoGIyE9FZIOIvCoiD4jIMLvXVApE5DwRWSciERGpqLRDEZktIv8SkTdF5Lt2r6cUiMidIrJdRNbavZZSISLjROQJEXk99l39ut1rsgM1FM5kBTDFGHMU8G/gSpvXUyrWAudQYeNtRcQN/Ar4GPBB4HwR+aC9qyoJvwVm272IEhMCvm2M+SBwPPDVCv23zIgaCgdijPmbMSYU+/UFYKyd6ykVxpj1xph/2b2OEjADeNMY87YxJgDcB5xt85qKjjHmaWCn3esoJcaY94wxL8d+3gusB8bYu6ryo4bC+XwZWG73IpScGANsSvh9M1V4c6k0RKQZmAastHkpZUebAtqEiDwGjLJ46SpjzEOxfa4i6vr+vpxrKybZXKeiOB0ROQC4H/iGMWaP3espN2oobMIYc3qm10XkS8DHgdPMIC52Geg6K5QtwLiE38fGtimDEBHxEjUSvzfG/Nnu9diBhp4ciIjMBv4LOMsYM9i6pSrwEnC4iEwUER8wF1hm85qUPBARAZYA640xN9m9HrtQQ+FMFgEHAitEZI2I3Gb3gkqBiHxKRDYDM4G/isijdq+pGMQSES4DHiUqfv6vMWadvasqPiJyL/A88AER2Swi8+xeUwn4MPAF4NTY3+IaEZlj96LKjbbwUBRFUTKiHoWiKIqSETUUiqIoSkbUUCiKoigZUUOhKIqiZEQNhaIoipIRNRSKUgJiXUffEZHhsd8bYr83i8gjIrJLRP7P7nUqSjaooVCUEmCM2QTcCtwQ23QDsNgY0wr8lGhuvqIMCtRQKErp+DlwvIh8AzgRuBHAGPM4sNfGdSlKTmivJ0UpEcaYoIgsAB4BPmqMCdq9JkXJB/UoFKW0fAx4D5hi90IUJV/UUChKiRCRqcAZRCejfVNEDrZ3RYqSH2ooFKUExLqO3kp0fsFGogL2jfauSlHyQw2FopSGrwAbjTErYr//GpgsIqeIyDPAn4DTYl1Xz7RtlYqSBdo9VlEURcmIehSKoihKRtRQKIqiKBlRQ6EoiqJkRA2FoiiKkhE1FIqiKEpG1FAoiqIoGVFDoSiKomTk/wcRbytLrgQKGgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "def draw_line(ax, v, p0, rect, N=50, label=None, color=\"black\"):\n", - " x1, x2, y1, y2 = rect\n", - " v = v / numpy.linalg.norm(v) * (x2 - x1)\n", - " points = [p0 + v * ((i * 2. / N - 2) + (x1 - p0[0]) / v[0]) for i in range(0, N * 4 + 1)]\n", - " arr = numpy.vstack(points)\n", - " arr = arr[arr[:, 0] >= x1]\n", - " arr = arr[arr[:, 0] <= x2]\n", - " arr = arr[arr[:, 1] >= y1]\n", - " arr = arr[arr[:, 1] <= y2]\n", - " ax.plot(arr[:, 0], arr[:, 1], '.', label=label, color=color)\n", - "\n", - "def zones(ax, model, X):\n", - " r = (X[:, 0].min(), X[:, 0].max(), X[:, 1].min(), X[:, 1].max())\n", - " h = .02 # step size in the mesh\n", - " xx, yy = numpy.meshgrid(numpy.arange(r[0], r[1], h), numpy.arange(r[2], r[3], h))\n", - " Z = model.predict(numpy.c_[xx.ravel(), yy.ravel()])\n", - " Z = Z.reshape(xx.shape)\n", - " return ax.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)\n", - "\n", - "fig, ax = plt.subplots(1, 1)\n", - "zones(ax, model, X)\n", - "df[df.y == 0].plot.scatter(x=\"X1\", y=\"X2\", color=\"blue\", label=\"y=0\", ax=ax)\n", - "df[df.y == 1].plot.scatter(x=\"X1\", y=\"X2\", color=\"red\", label=\"y=1\", ax=ax);\n", - "rect = (df.X1.min(), df.X1.max(), df.X2.min(), df.X2.max())\n", - "draw_line(ax, model.centers_[1] - model.centers_[0], model.centers_[0],\n", - " rect, N=100, label=\"hyperplan\", color=\"green\")\n", - "ax.legend();" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Conversion to ONNX = second implementation\n", - "\n", - "The conversion fails as expected because there is no registered converter for this new model." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MissingShapeCalculator\n", - "---\n", - "Unable to find a shape calculator for type ''.\n", - "It usually means the pipeline being converted contains a\n", - "transformer or a predictor with no corresponding converter\n", - "implemented in sklearn-onnx. If the converted is implemented\n", - "in another library, you need to register\n", - "the converted so that it can be used by sklearn-onnx (function\n", - "update_registered_converter). If the model is not yet covered\n", - "by sklearn-onnx, you may raise an issue to\n", - "https://github.com/onnx/sklearn-onnx/issues\n", - "to get the converter implemented or even contribute to the\n", - "project. If the model is a custom model, a new converter must\n", - "be implemented. Examples can be found in the gallery.\n", - "\n" - ] - } - ], - "source": [ - "from skl2onnx import to_onnx\n", - "one_row = X_train[:1].astype(numpy.float32)\n", - "try:\n", - " to_onnx(model, one_row)\n", - "except Exception as e:\n", - " print(e.__class__.__name__)\n", - " print(\"---\")\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Writing a converter means implementing the prediction methods with ONNX operators. That's very similar to learning a new mathematical language even if this language is very close to *numpy*. Instead of having a second implementation of the predictions, why not having a single one based on ONNX? That way the conversion to ONNX would be obvious. Well do you know ONNX operators? Not really... Why not using then numpy functions implemented with ONNX operators? Ok! But how?" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## A single implementation with ONNX operators" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file diff --git a/_doc/notebooks/numpy_api_onnx_ccl.ipynb b/_doc/notebooks/numpy_api_onnx_ccl.ipynb new file mode 100644 index 000000000..cc51aab6c --- /dev/null +++ b/_doc/notebooks/numpy_api_onnx_ccl.ipynb @@ -0,0 +1,622 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to a numpy API for ONNX: CustomClassifier\n", + "\n", + "This notebook shows how to write python classifier using similar functions as numpy offers and get a class which can be inserted into a pipeline and still be converted into ONNX." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
run previous cell, wait for 2 seconds
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from jyquickhelper import add_notebook_menu\n", + "add_notebook_menu()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext mlprodict" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A custom binary classifier\n", + "\n", + "Let's imagine a classifier not that simple about simple but not that complex about predictions. It does the following:\n", + "* compute the barycenters of both classes,\n", + "* determine an hyperplan containing the two barycenters of the clusters,\n", + "* train a logistic regression on both sides.\n", + "\n", + "Some data first..." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAoaElEQVR4nO2de3Ad9ZXnvwe97kUPIPYlm8UPwTibgrAUxnIyqZANDIawM6khUOGh2qGSsRhgapyAd5MJCSlndxhSk0oKMENtBLWinJmJlCxxNpmaZaPgkN0ZapeVZUxeMpnyghWUSnKvDXZsI9mSdfaPvm3dR/e93X378evb309Vl6RWP06/zvn9zjm/8xNVBSGEkOxxTtICEEIISQYaAEIIySg0AIQQklFoAAghJKPQABBCSEbpTFoAP6xevVoHBweTFoMQQlLFvn37DqtqoXZ9qgzA4OAgpqenkxaDEEJShYjMOq2nC4gQQjIKDQAhhGQUGgBCCMkoqYoBEEJIKywuLmJubg4LCwtJixIJuVwOa9asQVdXl6ftaQAIIZlhbm4O/f39GBwchIgkLU6oqCqOHDmCubk5XHzxxZ72oQuIEGIupRKwd6/1MwQWFhawatWqtlP+ACAiWLVqla/eDQ0AIcRMJiaA9euB66+3fk5MhHLYdlT+Nn6vjQaAEGIepRIwMgLMzwPHjlk/R0ZC6wkQCxoAQoh5HDoEdHdXr+vqstZnkNdeew3vfe97sWHDBtx+++04ffp0KMelASCEmMfgIFCr5BYXrfUZ5DOf+Qy2b9+OgwcP4oILLsDY2Fgox6UBIISYR6EAjI0B+TwwMGD9HBuz1sdMmHHoHTt24LHHHjv794MPPoidO3c23EdV8fzzz+OjH/0oAOBjH/sYvvOd77QuDJgGSggxleFhYMsWy+0zOJiI8p+YsEIP3d1Wh2RszBIrKFu3bsUtt9yC+++/H8vLy/jGN76B559/HldeeaXj9uPj47jwwgtx/vnno7PTUtdr1qzBL3/5y+BCVEADQAgxl0IhEcUPVMeh5+etdSMjlk0KKtLg4CBWrVqF/fv34ze/+Q02btyI9evX4+WXX3bd5/Dhw8FO5gEaAEIIccCOQ9vKH1iJQ7dik+666y7s2rULv/71r7F161YcP34cH/jABxy3HR8fx6WXXoqjR49iaWkJnZ2dmJubw0UXXRRcgApoAAghxIGo4tA333wzduzYgcXFRYyPj6Ojo6NhDwAArr32WnzrW9/CHXfcga997Wu46aabWhOiDIPAhBDiQFRx6O7ublx77bW47bbb0NHR4WmfL33pS3jkkUewYcMGHDlyBCMjI60JUYY9AEIIcSGKOPTy8jJefPFFPPPMM573ueSSSzA1NdX6yWtgD4AQQhpQKACbN4ej/GdmZrBhwwZcd911eOc739n6AVskEz2AUinRTDJCCAEAXHbZZXj11VeTFuMsbd8DiKieFCGEpJ62NgCsJ0UIIe60tQFgPSlCCHGnrQ0A60kRQog7bW0ADKonRQghgXniiSewYcMGiEiopSHaPgvIgHpShBDSEu9///vx4Q9/GNdcc02ox217AwAkWk+KEJJ2Qswj37FjB972trfh/vvvB2CVg77wwgtx3333Ndxv48aNLZ3XjUwYAEIICUTI9aCDlIO+7LLLAp+vGTQAhBDiRAT1oIOUg46SxAyAiKwF8DcA3g5AATylqo2nxiGEkLiIqB6033LQ7doDWALwH1T1JRHpB7BPRJ5T1ZkEZSLEH6wz0r5ElEcepBx0VCSWBqqqv1LVl8q/HwdwAEA4sxwQEgesM9LeRJRHHqQc9OOPP441a9Zgbm4OV1xxBe66666WZLARVQ3lQC0JITII4B8BXK6qv635390A7gaAdevWbZqdnY1fQEJqKZUspV/pHsjngdlZ9gQM5sCBA7j00kv97RRyL295eRlXXXUVnnnmmUgqgjpdo4jsU9Wh2m0THwgmIn0AdgO4v1b5A4CqPqWqQ6o6VOCHRUyBdUayQ4j1oFkOugIR6YKl/L+uqt9OUhZCfME6IyQALAddRkQEwBiAA6r6SFJyEBII1hlJLSa4vaPC77Ul2QN4P4A7AfxERF4ur/ucqj6bnEiE+IB1RlJHLpfDkSNHsGrVKlht0PZBVXHkyBHkcjnP+yRmAFT1BQDt9QRI9mCdkVRhZ9KU2nRSkFwuhzVr1njeniOBCSGZoaurCxdffHHSYhhD4llAhJCEKJWAvXs5RV6GoQEwDH6TJBY4iI2ABsAo+E2SWOBk2aQMDYAh8JskscFBbKQMDYAh8JskscFBbKQMDYAh8JtsQ0wN6HAQGylDA2AI/CbbDNMDOsPDVuG6PXusny3MckXSixHVQL0yNDSk09PTSYsRKSwv3wawUigxDLdqoBwIZhgcWNoGRDSTFCFhQxcQIWHDgA5JCTQAhIQNAzokJdAFREgUsFIoSQE0AIREBQM6xHDoAmoTTE05J4SYCw1AG2B6yjkhxExoAFIOawgRQoJCA5ByWEOIEBIUGoCUw5RzQkhQaABSDlPOCSFBYRpoG8CU83hhvSbSLrAH0CYUCsDmzVRIURN7xhXze0mE0AAQ4pHYM66Y30sihgaAEI/EmnHF/F4SAzQAhHgk1owr5veSGKABIMQjsWZcMb+XxAANACE+iG0mReb3khhgGighPomtyCfze0nE0AAQEid+BxGwpDSJELqASKxkOq097LTOTN9MEgY0ACQ2Mp3WHnZaZ6ZvJgkLGgASC5GmtaehJRxmWifHCJCQoAEggfCrcyNLa09LSzjMtE63m7l/v/mGkBgFDQDxTRCdG0lae5pawmGmdTrdzIUF4KabzDeExChoAIgvgurcSNLa0zZaNqxBBE43U9UyAqYbQmIUTAMlvrB17vz8yjpb5zZT5qGntadxtGxYaZ2VN/PNN4HbbrOUv43Xh0IyTaI9ABF5WkSKIvLTJOUg3mlV54Zatjrro2Xtm7lxY/oMITGCpF1AuwDcmLAMxAfG6dzYajMYjHEPhaQFUdVkBRAZBPAPqnp5s22HhoZ0eno6eqFIUzI3K1YaLjgNMpJEEJF9qjpUuz7pHkBTRORuEZkWkekSg1rGENSVY1LKvmdZ0pJqymnhiE+MNwCq+pSqDqnqUIEvdjopa9pvP1kyRo961ukxpJqaZBRJtjDeAJCUU9a0y9ddjxvvXY8/nJ9IPFPRl06PONU0LZ0L0p7QAJDoqNC05xw/hnMxj6cxgtWwNG1SKftuOn1uv0NTPMJU0zSNYyPtSdJpoBMA/g+Ad4nInIiMJCkPCRkHTbuILgzikPV7DJmKpRKw//slvPn9FcXupNM/Mj+BKz/i0BSPMMMmbePYSPuRqAFQ1WFVfYeqdqnqGlUdS1IeEjIOmrYLizjSNxhLpuLEBPCpiybwrg+txzkfuh5LayzFXqvT1+ZKGJMRiFtTPKJU0zSOYyPtBV1AJDoqNO1y/wDO9OTxxpfH8M3nC5Gn7JdKwGe2lvDVxRGci3mch2PoPD0PLSv2Sp3+o+8eQme+SVM8ggwbpu/HACPsDaEBINEyPIyv/cUsPrCwBxs6Z/Gvdgzj4MHolJz9ve/fD/xOxyGcRrViP3POimK3dfoFGwcTa4pzHFuEMMLelMQHgvmBA8HSx5NPAvfeW70un7eUXdhG4Mkngfvus/zqS0vA+YslHFxaj3OxUrhI83mI08knJiy3T1eXpfzHxqiN00ypZCn9yqJVUb14KSC1A8FIeimVLIVcS2dn+IFO29CcOgUcP25990fOKeCezjG8hTyOYQBL3XmIm4/FhKY43RXhwQi7J1gNlESG/Q2eOlW9/vTpcL0rboampwe481vD+Dm2YBCH0LlxsHHrz2+lzjBLL9g9kO5u6wb57YGwDEQ1jLB7gj0AEhmDg5YrppadO8PVUU6NPcD6/jduBDbeUMAFN4RcIiFM/3KrAwLo666HEXZP0ACQyKj8Bvv6rBb56Chwzz3hnicuQ2N7aA4fCHkEVyvuCo4mc8cEt57h0AVEIiX0SWAcsA3NyAjQ0WH19Hfu9G9obC/KxX0lrD5RLXClh+ZfLxzCD87pRndFcLmlCVhacVe0MkNPFghrAp42hT0AEjlxFKm0G3vPPw+8/rp/5W97UUY/OIFzL1uP0x9ccafUNrJfOTWIpfkQ/cutuCvo6yYtQANA2oZWSlSPjAC98yX89bw1cKx73nKn6MgI5vaXqjw0h1HAn+XGcKYnRP/y8DCwbx/w+OPWT6/uijT7upn1lDg0ACTz2F6UQdQPHPvtfBde++Ghukb2N2UYb+4P0b88MQFs2mSlM23a5C+Qm0ZfNwPXRsCBYCRWQs1WDOlg9pih3vkSZlE9cOwt5HF793fw0Yc34k93FNzHiQWUpVSyqpBe+ZH1Vi0im3YetMRBWrHDgWAkESp7+aE2+p58Eli7FrjuOseD+fEu2F6Uk/kC/rR7ZeDYKXSjA0v4u9O34Y8+vx6/fnTCuZEd8MLs3f79LYfw2/kMDVriIC1zUNXULJs2bVKSHsbHVfN51fPOs352dakCK0s+r1osVu9TLKpOTdWvr2J0tPpANQerPe/4eM3+LicpFlUnJ1XX5oq6BZN6Ern6c0xOVu9XLFrrm12Ygwj2bqtR1JNwOMbMjCWn/bPJMVNDwHtGggNgWh10auJK3c9CA5AenL7x2mVgwNJrNk0Vt33gnp66g53p61edmmquWzycZNs21SFM6Zs4r17o3t7q/aamrGM1ujAHane7HeN6Enld7B2wjr9tm/XTvhj7d8ebkkLs5zAw0F7XZSg0ACRWnPRig0a790bh1JRqf3/dwebRo7tHi431sYeT2Js4tsqd9vPbmi33Pkozxbrd1uaK+sZkucXvZj3DaCl76mbFgClyZAA3A8AYAIkEp/T07m4gl3POVvTsFnYY9qsAPoGd+KPtBfT1NUiL93ASe5PDKGArrHjACfSiLlWicrCV1zTMiljB6k3r8dzIRNVuX3q6XLLixAnn2hauN8UHJmXfxDFAhDTGySqYurAHkC6cevlujT5fDenxcV3qyesx9Ok8evQujFa19MfHVbu7V47T1VX2MPjoAdj/Xo2i/kH3pC7nHOIBtbGARq1Zl3OXZor1uzXyn7XSA6DvPbOALiCSBH56+X7cwqWZol7dM6WrUTyry3I5K0Y7M2P97qjnPJzEcZPRUSv20N8fzGftN1ZgC2FfiN8YgNONDxivIOmHBoCkgqAGo6vLavWfd56lp2sbur29lnHwepKqTewT9fdbBx8d9S94kNa3fczaLKBm53ILdLMHkFloAEhbYqduNss4snWd72QTL0rTU/qShpP50uxczeRl9k0mcTMAHAlMzMbDCNu9e62Y5rFjK+vyeeDMmfqAsO8Bp04HHxiwyi5s3ux/VGsro5e9nKuZvK3KQFJJoJHAIjIgIr/jsP6KMIUjxBGPGStOGUcA8Ld/C/T2Vq/znUTTrNqm31GtrWS+eDmXl+qgzL4hZVwNgIjcBuAVALtF5Gcisrni37uiFoxkHB8TnbhlYl57LbC8XL3t6dPAm282LhFRVUaiWZqni8I93DcYXqFLW6CGOa5l0lwdlMSPk1+o7BZ6GcA7yr+/B5YxuLn89363/aJcGAPIEA4ZK2f6B/Rnu6aajbFydM/XBoprY6P2fq4u9tqDO+1U9qu/sG3cU0jAE7UC2SOEm/nwOciKVAC/QWAAP6n5+x0A9gH4JICX3PaLcqEBiBDTFIZDMPMk8npJf9G3UnULFOfzVkKP33pFjlaiwQjfwIk2bgHddqsNRCLHzQA0igEcr/T/q+qvAFwD4CYA7w6zF0ISxqTRoTYVrozl/gG8hTy2YgyvHi/4nva2UAAuuKDefd7RYZXfr/QyLS5Wb1PnzndzTQHA5s147UQhvEKXbj7/Eyfi8eFHMWELJ4ExikYG4F4AUrlCVY8DuBHAw1EKRWIkhknFA3/zw8M4vG8Wz27fg3f3zuKbWKnB7FepusVG3SouVG5TNbtik0BsXx+wsNDkGDW43p+Ip3ts+FyiaBSY2NDIOk7dAqvHgFcB/DmAjop1bwfwd3DpTkS90AUUARGPDvWaIt9oX4fab65ulUaerNoUeNv9U1n24X2dU7qmp+juYm+QZ28f308Bz6b3xyVvv1WPXcPzRjFgjIPQEsVNZzcyABcAGAXwEwC/B+A+ALMA/gzAOW77RbnQAERAhB9mK4d2K4fT1+euVL0Ym1rFae/zxzmrHPOp/Hm6nM/rwYfGrW2aRZbLJ3KSt6fHcte3fH/sIEZ5LoJWjKqn80bRKGAZikTxbQDObmAp/mUAcwDWNNs+yoUGICIiGh3ayjfvtG9/v+quXe4t/6DGpjRT1KUelwhxLmfVkcjlqu9LjWEIcq2e96nQ+Mv5vN7ZNd6SvW56XvYA2o4gPYDzATwJKx30BgCP2b0Bt32iXmgAIiSCLKCwewCN9m2pgelmbTo6qtd1ddUJUFmuJ0ipH6d9SjMVz8IlG6qyCJ7fhrSnexu0UeDHB8cyFLERxAC8CuBTADor1l0J4H8DmHDbL8qFBiBGQjIIrXzzfvZtqYFZLNaXD62sJ125nK0oFzxFv9E17r51XJcrD/rQQ3XG6SgGdAhTvgyNBy+Wtx29XIwfHxyJhSAGwNXdA+BP3P4X5UIDEBOtOplr8PPNNxpv5VVs38bGaQKBBx5oaABaTdGvvK5i0dLza3pc5gauMU6L3Xldm2sQqHa4J06PMlRd3MgCh3UiGo/ABI4BmLTQAMRAgr7aMOyOrSOq3CjNdnDT5LW9gO7ulnz+tddpV5f+8petvx3nIR4YsKyDQ9C52eXF+ijdbogte6uNiZAbJVnDSAMAa0zBzwEcBPBAs+1pAGIgoWyNUJWVH2XR6Hrt49ROBN+CvG7ZTbmcyzzELbSiay9tNYp6Te+UvjTZ+BiBGtpOF5bL6XIuhIfKAHLLGGcAAHQA+H8ALgHQDeBHAC5rtA8NQED8+mAS+NhCszt+5W+2fcW9c0sh9eNycpnT/uxyO6x01KMYsGIBDQ7a7LFWXtodZ497XsPjttTQrrkhL9/6kB516tH4fahMIW0ZEw3A+wBMVvz9WQCfbbQPDUAAgnzRCWRrhGZ3fCqLYlH14EPlwKuHaSIbFZLzep09PfWKP5ez1g8MqK7NFXX3A1ZL3e24fuagWZtr0LOoka3lZ1C+IaWZouN5l9kDSAQTDcBHAfyXir/vBPCEw3Z3A5gGML1u3brIblBb0moeZswBt1Dsjo9gZKUSXZsr6vcecr7esPXP6Gi9AbDDDi9NWsp/ba4YeNKvWt6YnNLF3vOaGsUwG9r2sSp7NCdhDbCzr8HX68UU0pZIrQGoXNgD8EkKu86h2B0nZVHTZD42Ot48B79MFLfRnmO+amTzuNUTOYZ+nUeP3oVRR+XuJE9fn/sgOa8WI0xDV3ms1SjqECyjViy24GZiFlBgTDQAdAFFTZa7zrV5ljX3YanHKi1deWv+ODdujQiu0UxR3cYqfeZwkmVA78JonbFxCyT39zdQqB5b0GE2tJ2OleVXMklMNACd5cFmF1cEgd/daB8agACw6+w6uczVPSuDqRpm4GgMt3FqSk/l6qPD8+g523KuxJanr6/eELgqVI8t6DAb2rXHMrpT2sY9DOMMgCUTfh/AP5ezgR5stn3bG4CoXsBWj5vGD6NJD0Dzed09Wjyr1K/umdJT+fMaaqYob2NppqjzqI8OH0O/FZuoZGZGddcuPfzCjO7aVZ9V5Euheh1QENLzN7YH0ObjDIw0AH6XtjYApr6ApsrVCCeZm5RVLs1Eq5ma3capKdVP5kZ1ucYALPXUyLBtW9X/T27d5jXmXXW9xaIHobwI3sK9MKZTaqxVCg8aAJMx9QX0IpdpvQMfWUB1RKSZvN7GfF71LozqPHr0OM7Vt2AFq88yM1N9kPIy8YUZ7empjgE0soF21tNit0ehIngvjXptjPZLhYObAWg0IxiJiyazTCVGM7n8zvAUx3SALjK/uf8Q9h4qoDTYYCrF4WFgdhbYs8f6OTzsvF0Zr5fj5fHaM2Ce7hrAMgSAoKtLMTBQsdPUVN2xFcAPvjiF7m5r8rBHHwW2bHGe5G3r1pV1b184hJOnmwgV4XtZKMQzq6UnQpp5LZWzXTpZBVMX9gAMkssls6Y006R1HbUryUEuu3hamKf2fDnFor4xaaVANn28xaI1UMptQ4cewDKg78JM1eaTk/UN2t5ea2kU9F7O56sHn5n6XkZBi70/0z2loAvIcIxzjJZxk8uh23wUVmZNnehxK5IKmcOYQKUWz5dToRUWuy05Gj5eL66IihjAMqBf7dpWt/nkpLN8tRWv7+xaGQFty1enwGpiDrptW/AbZzoB/VJpsJM0AGnAKMdoBW7RxJq33p6opO7lT8LHWpb5pcliMrMbOuX117awnWT2okkqsoDy+ZWBVpX3vtFYuCoj1KCHEnVgvF1IQwiBBqCdScpwjFsDp+xh/rdj3LMyjEuRRHFqT8cMqhV89gSf27pS5O0k8vrCtuqKpQ2zgJqI+rNdKdBsBsAeAA1AciTsfCzNFPXqnqmqKQobukMScHFFceqmx2ygFZraa48GffeotyJvzQ7vJqrJPQDTOsumenBtaADaEUOaHp5f/gS/2ihO3fSYDjcmLHtdLFqD12onkDnT37iF7nZ+12dooGYzNeBqmlGqxM0AiPW/dDA0NKTT09NJi2EOe/daKZjHjq2sGxiw0hg3b45VlFLJyg4cHDQktc8ASiVgbn8JgziECzYOooQC1q+3UjFt8nkr49TvPdu7F7jjuhJ+cnw9zsXKAc/05NHxuvMBSyU0PL/rMzTo4Ta7BuKMiOxT1aHa9RwHkGZCyl8OA6PyukMkaG63PUTi2tsKuOgjmzGxpxBqWv3gIPCrpQK2YgxvIY9jGMBbyOPkzjHXh+B0/o4O4NlnretzfYYGPVxTh8ykFRqANGOPHsrnrZZ/Pm/9bcCH2jIGjKrxO87NplRyHojV1+dur/1erv3o/z4/jMv7ZvEH3Xsw+sAsTt3iPnjNqb1w4gTwiU/4u74kMajN0x44+YVMXRgDcMFk52MQ/Dp5y2mROjMTmgithFe8TDPcYKoCXz7tYtHfvOvNqojOzJj/KhkYljAeMAhMYiGMkpl+NG9EA5Vaye1udgkeCpV6vn0zM/VTTDbbf2ZGdceO6pHBwMq0lKYFV51otzZP1LgZALqASGtU+i6C+kwq8ePkPXAAeOKJ6nVPPGGtb5FWXA3NPHOVLvVWfNoTE8DGjcCpU973n5gAPnRVCf/rK3uRP1ntb1pYsI5V6bYyta6NQWGJVEMDQIJTqfDXrQM+/vF6x7dfDeJH8zoUR2u43gethle81pULamjsOEOt8m+0f6kEfO9jE3hlYT2+89b1mMV63I4J9PUBPT3WNVbS2Qn84BslvPl9b8EJA8I2xC9O3QJTF7qADMLJd1G7BB016tXJ61IeOexYQNSuhiA+bScXFWC5cNz2f2myfuDYSeR14vGizszUP847sDLKeLG7sWBh5ObTrRMdYAyAhIqbBvLqjG72tXvVBm1SrMyv8nOyvz09jW3fG5NTerRm4NhRDOgbk5aRtpV4T497tVAnAcMYj2jq4K52gQaAhIvTV9/V5a0pG/bXHkEWUBrw0nOoMizF+klgFrurNfXMjGp3t+oQ6kcZL/Y69+haLYaW6ID2jHQ7aABIY4J8CE4ayEvL3oDyFe1CbUZR5a13tLPjVgnoxV6rVHat1ZiasmYWc+sBlGaKXgrD+nqkiVXTzFC3gwaAuNNqIrofw5GG2rkpw2kswOhoA6Xc4JlVKvPbz8YABnSxy6o06vaatJKbn0ibIGMNERoA4kzcH0LGPryoGR+vn+jFjgf09wezs7Yy7+9Xvai7qLsfmNLSTLHpY2vFmxL74K6MNUTcDEBnkhlIxADsRPTK6lp2InkUSdZ2fuXIiJVnaE9ky4Ru39ipoAsL9f/r6go+jmF42JpX2Kr/VkChUMBf/mX1K2Kfo/I1KRSCP8bqc8bwOrCmBACOAyBJfAjDw5bSP33aMj7bt5tRiCZliexOg8hszpwBdu4MPo6hcqBVqQQ8/HD9NouLVn2jsG6Z18FdoTymdq6j5QenboGpC11AERF3/9tEN1AKA4JuQzEqxQ8jyWVysr5sBKB6663xhY5sQn9MzAJKXrF7XTJjAJJ4KeM8p2n+VxMNkkcqbXcuZwWDwxTbPr5TjCHoLQuqxFP8mBLHzQDQBWQaYdTTCUKcxVVM87+muMh8ZcmJX/wC+Pznw3uElWWtK8nlrPMEuWVupbK9uHNS/JhaJirvJA2ASbTydaQJ0/yvphkkn0Rlu50Ua3c38N3vAvfcE+yW+VbiFZov5Y8pMFG2CWkATCJLTRyv1dLiwDSDFBF+W5F9ffWt/9OngR//OPgt86XEazRfYc9E3TkffdT6PGJrI8WcKBB5m9DJL2Tq0vYxADo5k6WNA4JB/O5TU/VzDdj+f6e5DfzK0jDnoMG3YJ/THuwWW9w+gUSBsMJlYBA4JXC6IxIyQdsVxaKzAejvbz1e39RwNNF8sbeVEmqchXVaNwNAF1DU+O0ymuQaIf4xcCxBM8+im8iFgjWWoJalpdb97k3jFk18RbF7SxNyz0bunXSyCqYuqesBpDC3nLSAoc+7USvSi8ijoyulJWK9rAa94az0ACpP34p3EnQBxQz9+dnC8Oddq0t3jxb1jckpXZsrehI5sfBIgxPH7i1NsXvWzQCwFlBUxF1jx41SKcYCKxnGlOftQmWtnXe9NIGB7SNYOqcbryycxlaM4ZuwXI1uIrdS56clGpw49vpBsZ8wemgAosKEpOWJCStnrLvbkmVsjDGFqDDheTehUAAKKAEftPIKOzGPTgBPYwQ/wBYcRsE0kZsSu2FKzBJGA4PAUZF0bnlWBpWZQtLP2ysOwcxFdOHy3kPGikyiI5EegIjcCuA/ArgUwHtUdToJOSInyS6j4S6JtiQNLgKHnspAfhGPfHsQazaaKTKJjqRcQD8FcAuAJxM6f7g08rMn1WVMgUuiLTHdRVA5H0NXF7C4CBkbw8YbDJaZREYiLiBVPaCqP0/i3KGTVPG2ZqTFJUHip13Gmhg45iJtiJUhlNDJRf4ngE81cgGJyN0A7gaAdevWbZqdnY1JOg+USpbSr3Sz5PPWR2WKomUWEGlHmODgCxHZp6pDtesj6wGIyB4R+anDcpOf46jqU6o6pKpDBdMUWBqKt8VZ5pmQOGCCQ2hEFgNQ1S1RHdsY6GcnJH6Y4BAaTANtBfrZCVkhLp88G16hkYgBEJGbRWQOwPsA/HcRmUxCjlBol4AaIa0QZzIEG16hkWgQ2C9DQ0M6Pd2eQwYISS1JJUMwwcEzbkFgloIghLRGUj5508dcpADGANoZLz5Z5lKTVvHpk+crZw40AO2KF5+sqYPYSLrw4ZPnK2cWjAG0I158smkYxEbSRROfPF+55Ih9IBhJEC8D1NIwiI2kiyaDDk175eiKogFoT7z4ZJlLTWLGpFeOrigLGoB2xItPlrnUJGZMeeUaVZLIWq+AMYB2xkueNHOpScwk/crt3Wu1/I8dW1k3MAB8+tPAF7/YnvXl3GIANACEkEzhFoxWBRYWqte1S4CaQeCoyFqfkZCUU+mK6u21fn7uc0BPT/V2WciJoAFoBUaSCEkttvND1TIKpgSo44QGICisSU5I6MTRobY/3YUF4ORJ6+f27cCjjyYfoI4bGoCgmJbUTEjKiatD7fbpXnVV9gr7shhcUExKaiYk5VR2qO3g7MgIsGVLOK3wysyjRp9u1urLsQcQFFOSmglpA6LsUNf2LPbs4adrwzTQVkk6qZmQNiCqOkGNjgtk59NlGmhUcNJ1Qlomqg51o54FP13GAAghhjA8bPn8w2yVM1TXGPYACCHGEHarnKG6xrAHQAhpa6LoWbQLNACEkLYna+mdXqELiBBCMgoNACGEZBQaAEIIiRhTiwbTABBCSISYXDSYBoAQQiLC9KLBNACEEBIRphcNpgEghJCIMH0kMg1AUEyN6hBCjMH0kcg0AEEwOapDCDGK4WFzJ5phOWi/RFW3lhBCIoLloMPC9KgOIYR4hLWAvGJP/NLXZ3ZUhxBCPMIegBcqff6bNlmJvKZGdQghxCPsATTDabbqsTFg3z7gxAnWlyWEpJZEegAi8mUReUVEfiwi/01Ezk9CDk+4+fxPnOB8coSQVJOUC+g5AJer6hUA/hnAZxOSozmmj+QghJCAJGIAVPX7qrpU/vNFAGuSkMMTpo/kIISQgJgQA9gK4JtJC9EQzilHCGlDIjMAIrIHwL9w+NeDqvrd8jYPAlgC8PUGx7kbwN0AsG7duggk9QjnlCOEtBmRGQBV3dLo/yLycQAfBnCdNhiOrKpPAXgKsEYChykjIYRkmURcQCJyI4A/B/BBVX0rCRkIISTrJJUF9ASAfgDPicjLIjKakByEEJJZEukBqOqGJM5LCCFkBZaCIISQjJKqctAiUgIwm6AIqwEcTvD8YdEO18FrMANegxk0u4b1qlqXxpgqA5A0IjLtVFM7bbTDdfAazIDXYAZBr4EuIEIIySg0AIQQklFoAPzxVNIChEQ7XAevwQx4DWYQ6BoYAyCEkIzCHgAhhGQUGgBCCMkoNAA+EZGHyjOZvSwi3xeRf5m0TH5J1YxsLojIrSLyMxFZFpFUpfCJyI0i8nMROSgiDyQtTxBE5GkRKYrIT5OWJQgislZEfigiM+X36L6kZfKLiOREZEpEflS+hv/k+xiMAfhDRAZU9bfl3z8J4DJVvTdhsXwhIjcAeF5Vl0TkSwCgqp9JWCxfiMilAJYBPAngU6o6nbBInhCRDliz4F0PYA7AXgDDqjqTqGA+EZF/A+AEgL9R1cuTlscvIvIOAO9Q1ZdEpB/APgAfSdNzEBEB0KuqJ0SkC8ALAO5T1Re9HoM9AJ/Yyr9ML4DUWdBUzcjmgqoeUNWfJy1HAN4D4KCqvqqqpwF8A8BNCcvkG1X9RwBvJC1HUFT1V6r6Uvn34wAOALgoWan8oRYnyn92lRdf+ogGIAAi8rCIvA7g3wHYkbQ8LbIVwP9IWogMcRGA1yv+nkPKFE+7ISKDADYC+L8Ji+IbEekQkZcBFAE8p6q+roEGwAER2SMiP3VYbgIAVX1QVdfCmslsW7LSOtPsGsrbNJ2RLUm8XAMhrSAifQB2A7i/pnefClT1jKpeCasX/x4R8eWOM2FOYONoNptZBV8H8CyAL0QoTiDCmpEtSXw8hzTxSwBrK/5eU15HYqbsN98N4Ouq+u2k5WkFVT0qIj8EcCMAz4F59gB8IiLvrPjzJgCvJCVLUCpmZPtDzsgWO3sBvFNELhaRbgB3APj7hGXKHOUA6hiAA6r6SNLyBEFECnYGn4jkYSUW+NJHzALyiYjsBvAuWBkoswDuVdVUteBE5CCAHgBHyqteTGEm080A/hpAAcBRAC+r6ocSFcojIvL7AB4D0AHgaVV9OFmJ/CMiEwCugVWG+DcAvqCqY4kK5QMRuRrAPwH4CaxvGQA+p6rPJieVP0TkCgBfg/UenQPgv6rqX/g6Bg0AIYRkE7qACCEko9AAEEJIRqEBIISQjEIDQAghGYUGgBBCMgoNACE+KFeRfE1E3lb++4Ly34Mi8j0ROSoi/5C0nIR4gQaAEB+o6usAvgrgr8qr/grAU6p6CMCXAdyZkGiE+IYGgBD/PArgd0XkfgBXA/gKAKjqDwAcT1AuQnzBWkCE+ERVF0Xk0wC+B+AGVV1MWiZCgsAeACHB+LcAfgUgdZOhEGJDA0CIT0TkSliFt34XwPby7FKEpA4aAEJ8UK4i+VVY9eN/ASvw+5VkpSIkGDQAhPjjTwD8QlWfK//9nwFcKiIfFJF/AvAMgOtEZE5EUlGdlGQXVgMlhJCMwh4AIYRkFBoAQgjJKDQAhBCSUWgACCEko9AAEEJIRqEBIISQjEIDQAghGeX/A5vQ4dfeS5N/AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from sklearn.datasets import make_classification\n", + "from pandas import DataFrame\n", + "\n", + "X, y = make_classification(200, n_classes=2, n_features=2, n_informative=2,\n", + " n_redundant=0, n_clusters_per_class=2, hypercube=False)\n", + "\n", + "df = DataFrame(X)\n", + "df.columns = ['X1', 'X2']\n", + "df['y'] = y\n", + "ax = df[df.y == 0].plot.scatter(x=\"X1\", y=\"X2\", color=\"blue\", label=\"y=0\")\n", + "df[df.y == 1].plot.scatter(x=\"X1\", y=\"X2\", color=\"red\", label=\"y=1\", ax=ax);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Split into train and test as usual." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The model..." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0,\n", + " 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0,\n", + " 0, 0, 0, 0, 0, 1], dtype=int64)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy\n", + "from sklearn.base import ClassifierMixin, BaseEstimator\n", + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "class TwoLogisticRegression(ClassifierMixin, BaseEstimator):\n", + " \n", + " def __init__(self):\n", + " ClassifierMixin.__init__(self)\n", + " BaseEstimator.__init__(self)\n", + " \n", + " def fit(self, X, y, sample_weights=None):\n", + " if sample_weights is not None:\n", + " raise NotImplementedError(\"weighted sample not implemented in this example.\")\n", + " \n", + " # Barycenters\n", + " self.weights_ = numpy.array([(y==0).sum(), (y==1).sum()])\n", + " p1 = X[y==0].sum(axis=0) / self.weights_[0]\n", + " p2 = X[y==1].sum(axis=0) / self.weights_[1]\n", + " self.centers_ = numpy.vstack([p1, p2])\n", + " \n", + " # A vector orthogonal\n", + " v = p2 - p1\n", + " v /= numpy.linalg.norm(v)\n", + " x = numpy.random.randn(X.shape[1])\n", + " x -= x.dot(v) * v\n", + " x /= numpy.linalg.norm(x)\n", + " self.hyperplan_ = x.reshape((-1, 1))\n", + " \n", + " # sign\n", + " sign = ((X - p1) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel()\n", + " \n", + " # Trains models\n", + " self.lr0_ = LogisticRegression().fit(X[sign == 0], y[sign == 0])\n", + " self.lr1_ = LogisticRegression().fit(X[sign == 1], y[sign == 1])\n", + "\n", + " return self\n", + " \n", + " def predict_proba(self, X):\n", + " sign = self.predict_side(X).reshape((-1, 1))\n", + " prob0 = self.lr0_.predict_proba(X)\n", + " prob1 = self.lr1_.predict_proba(X)\n", + " prob = prob1 * sign - prob0 * (sign - 1)\n", + " return prob\n", + " \n", + " def predict(self, X):\n", + " prob = self.predict_proba(X)\n", + " return prob.argmax(axis=1)\n", + "\n", + " def predict_side(self, X):\n", + " return ((X - self.centers_[0]) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel()\n", + " \n", + " \n", + "model = TwoLogisticRegression()\n", + "model.fit(X_train, y_train)\n", + "model.predict(X_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's compare the model a single logistic regression. It shouuld be better. The same logistic regression applied on both sides is equivalent a single logistic regression and both half logistic regression is better on its side." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.6, 0.72)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "lr = LogisticRegression().fit(X_train, y_train)\n", + "accuracy_score(y_test, lr.predict(X_test)), accuracy_score(y_test, model.predict(X_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, this is true in average but not necessarily true for one particular datasets. But that's not the point of this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[ 0.03522525, 0.03404299],\n", + " [-0.29776838, 0.07071687]])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.centers_" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.10947197],\n", + " [0.99398988]])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.hyperplan_" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([[-0.90862235, -0.21881959]]), array([[1.14845026, 1.08707429]]))" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.lr0_.coef_, model.lr1_.coef_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's draw the model predictions. Colored zones indicates the predicted class, green line indicates the hyperplan splitting the features into two. A different logistic regression is applied on each side." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + ":20: MatplotlibDeprecationWarning: shading='flat' when X and Y have the same dimensions as C is deprecated since 3.3. Either specify the corners of the quadrilaterals with X and Y, or pass shading='auto', 'nearest' or 'gouraud', or set rcParams['pcolor.shading']. This will become an error two minor releases later.\n", + " return ax.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABE2klEQVR4nO2deXxU5bn4v+9s2SAQQgTCFsQFhSIIGkBELVCRa8Fqa1Fr2yuuty5XK23VX0Vs9Wpxa/W2FcV73W3rArQCitSlXhYVpBaEVtFAMCwhDASyzXLe3x+TGWYmM8msObM838/HD+bMzDnPmZy8z/vsSmuNIAiCkH9YzBZAEARBMAdRAIIgCHmKKABBEIQ8RRSAIAhCniIKQBAEIU+xmS1APPTt21dXVVWZLYYgCEJWsWHDhv1a64rw41mlAKqqqvjoo4/MFkMQBCGrUErtiHRcXECCIAh5iigAQRCEPEUUgCAIQp6SVTEAQRByA7fbza5du2htbTVblJyisLCQQYMGYbfbY3q/KABBELqdXbt20bNnT6qqqlBKmS1OTqC1pqGhgV27djFs2LCYPiMuIEEQup3W1lbKy8tl8U8hSinKy8vjsqpEAQiCYAoRF3+3G5qafP8KcROvQhUXkCAImUFDA+zYAUqB1jB0KJSXmy1VTiMWgCAI5uN2+xZ/wwCv1/fvjh1ptQRqamoYNWpU2s4fDz169DDlulllARxodvHix7vMFiOjKFk43WwRBCFuBl73GAe/8AZ+tnoMSrQRsiM1tEHTl1vx2tKzT23c9RVeVxsHv9iclvMH4/F4sNk6WW610akcvY9Nj6LKKgUgCEJuYlgUKmw4odK+434+qN/E+3s+YnL/8ZxeMSY11zW83HTbXXzw8SYG9DuGe+74CdfeejvvLvsjANu/3MEVN83j3WV/ZPSUc7lg5rm89e7fKCos5ImH7+fYqiHsbzjALT//BbvqdgNw7//7KRPGj+W+X/+WL3fWUrNzF4MqBzD1zEn8ZdVqGg8fYfeefVx8wfn89MbrQuQ50tTMZdfcyMHGRtxuN//vlhuYOf3r1NTUcN555zF58mTWrFnDwIEDWbp0KUVFRUndv7iAspymeavMFkEQkkZbFM097GjAUKDB93O7AvigfhMXrLqKezc9ygWrruKD+k0pue72mp1cefkc1q5cQq/Snnzy6VZKe/TgH59uA+D5V5Zw2UWzA+8v7dmDNSte46rLL+G2X94PwM9+cR/XXXE5f13yEk//9mFuun1+4P3//Gw7S559gsW//hUAG/++mWf++2HeX/4KS5a/ycefbAmRp7DAwbO/e4R3l/2RPz//FP/v3gfwj+397LPP+NGPfsSWLVvo3bs3r7zyStL3LxZADtA0b5W4goSsx+2w0tjbgsXQGBYVWPwB3t/zES6vCy8GLsPN+3s+SokVMHTQQL528ggAThl1MrW76rj8uxfy/MtLuOeOebz2+kpWv/pi4P3f/uZ5AFz0zfO4/R7fov7u/63nn59/EXjP4SNNHGlqBuC8aedQVFgYeO3sMybSp6w3AN88dyrrNmxk7OiRgde11vziwV+z5oMNWCwWdu/dx779DRT068mwYcMYM8Z3z+PGjaOmpibp+xcFkCOIEhByAW1ReC0dUxkn9x+Pw+rAZbhxWOxM7j8+JddzOByB/7darLR625g1Yzq/+s3vmTLxdMaMOjmwYENomqX//w1tsOqV5yksKOhw/uIwF014mqYi9Oc/LX2dhgYn7yz9A3a7ndFTzqWtrY0CoCDo/FarlZaWlrjvNxxxAeUQ4g4ScpXTK8awZPoT3H7K9SyZ/kTKYgCRKCwo4OtnTuLHd/6SSy+6IOS1V19fGfj3tLGnAHDO5IksevqFwHv87qNIvPN/a3EePERLayuvr/or1ePGhrzeePgIfcv7YLfb+dvaD6j9qi5FdxUZsQByDL8SEGtAyDVOrxiT1oU/mO/M/jdeX7War585KeT4wUONnDHzQgocDp58xOcCuv/O25g3/x7OmHkhXq+XiaeN4+Ff3hnxvKeOHsX3/+Nm6vbs5eILzg9x//ive8nV1zPpvG8x9msjOWF4bC0dEkX5AwzZwLEnj9b3PL/cbDGyClEEQiYy8LrHGD6wn9liROXRJ/6XxsOHueOWGwLHRk85l7eXvER5n7KEzvnCy0v4ePMWFt51R9yfjScNdOvWrZx00kkhx5RSG7TWHfxm4gLKccQtJAjx8b1rb+Kl15Zx7Q+/Z7YoaUcsgDxBLAEhk8h0CyDTEAsAaGtVNDqzSuSMQSwBQRDCyarVdPcOOzfO7M+alclVv+UrogQEQQgmqxSANsDVZmHRgjKxBBJElIAgCH6ychW12qC+zmq2GFmLKAFBECBLFYDXAxWV3q7fKAiCkGJ21O5i2oWXcuo5M7nihltxubJ3eE1WKQBlAUeBwdXznZSWGWaLk9WIFSAIiXHXrx7mun+/nI1vL6dXr1Ke/dOrZouUMFmlAAYMdfOb5XuYNCP5HhiCKAEhu9jfYGXjJ0Xsb0iN+/fehx/jd//zbODnXzzwG37/P891+hmtNe+t/YDZ5/nSqi+5cBbLV/01JfKYQVa1gigo1LLzTzHSRE7IBl5eVsqNtw3Cbte43YpH79vFRd9sTOqc3/vOt7j8P27mun+/HMMwePX1FSx9bjFnnv/tiO9/4uH7qSjvQ6+ePQPDXSr796duz76k5DCTrFIAQnoQJSBkMvsbrNx42yBaWi20tPqO3fCzQZw16Z/0LU88Fjhk0ED69O7FJ1u2sm9/A6NPPokhAyv5219ejvqZhgPOhK+XiYgCEABRAkLmsvMrB3a7Diz+ADabZudXDvqWJ+cOvvzii3jhlaXsq9/PZd++gMNHmpg55wcR3/vEw/dz4nHHcujw4cCIx7o9e6jsf0xSMpiJKAAhgHQSFTKRIQNduN2hffM9HsWQga6kz33+N6byX4/8N26PmyceuR+r1dqpBQBw5oTTWLpiFRd98zxefHUZ5007J2k5zMK0ILBSarBS6m2l1KdKqS1KqZvMkkUIRYLDQibRt9zLo/ftoqjQoGcPL0WFBo/etysp948fh8PO5AmnccHMc7FaYwsu3/WTm/ntU89w6jkzcR48yOXfuTBpOczCTAvAA/xYa71RKdUT2KCUWqW1/tREmYR2xCXkw9HqobjJTXOJHVehGMxmcdE3Gzlr0j/Z+ZWDIQNdKVn8AQzD4KNNn/C/jz4Y82eqhgxm9Wsvdv3GLMA0C0BrvVtrvbH9/w8DW4GBZskjdCTfLYHKmkNMW7adiW/XMm3Zdip3HDJbpLymb7mXU0e3pGzx3/bZdk79+kzOmlTN8GFDU3LObCMjtjRKqSpgLLA+wmtXA1cD9O0v+qG7yVdLwNHqYcwHe7B5NXh9LdPHrN/D/n4lYgnkCCOOH86md1aaLYapmF4IppTqAbwC/KfWukNir9Z6kdZ6vNZ6fM+yPt0voJCXlkBxkxsdNpxcWxTFTdlb9i8I4ZiqAJRSdnyL//Na6+ytp84D8k0JNJfYUUbosCRlaJpL7CZJJAipx8wsIAUsBrZqrR8ySw4hdvJJCbgKbWyq7o/HqnDbLXisik3V/cX9I+QUZj7NZwCXA/9QSm1qP3a71lpmPmYw+RQTqBvai/39SiQLSMhZzMwCel9rrbTWo7XWY9r/k8U/C8g3S+BgeZEs/kKARc+8wKnnzKRs+NeyvjWE6UFgITvJJyUgCMFMGDeWJc8+weCBlWaLkjSiAISEESWQHThaPfRuaMHR6jFblKRQDQewfrIZ1XAgJedLpB00wOiRJzFkUG6kpItdm0YanRbq66xUVHqljbVgCpU1hxjzwR60RaEMzabq/tQN7WW2WHFjX7ac4tvmg90Gbg/N9y3A/c2ZSZ0zkXbQI44fntQ1Mw1RAGlizYoiFt1dhtUOXjdcPd+Zk4Ns8ikonG3kSjGbajhA8W3zUa2t0N4RtPhn82mcNAFdnnhtUCLtoHON7HkKsohGp4VFd5fharNAm+/YogVljKpuy0lLQLqIZiaBYjbv0XoGfzFbNikAy1d1vp1/UDtobDYsX9XhTUIBQPztoMUCELqkvs6K1U5g8Qew2nzHc1EB+BFrILPIlWI2Y2AluMPiFx6P73iSJNIOOpeQIHAaqKj04g3rGOD1+I7nOvkUGM704GquFLPp8j4037cAXViI7tEDXVjo+znJ3T8k1g768f99npFnTKVuz14m/9tF3Hjb/KTlMIvsehKyhNIyg6vnO1m0oAyrzbf4Xz3fmdO7/2DywRLIluBqrhSzub85k8ZJE7B8VYcxsDIliz8k1g76mh9exjU/vCwl1zeb7HwasoBJM1oYVd2Wt1lAuawEsi246iq0ZaRc8aLL+yTt8w9m22fbmXPVjzj/G1OlHbSQekrLjLxb+IPJVSWQK8HVfEfaQUsMQEgzuRgTyJXgqqloA6111+8T4iLe71QUgJB2ck0J5Epw1Uxc9bU0trpFCaQQrTUNDQ0UFhbG/Bl5YoVuIdfcQbkSXDWLhuWPA9ewv2IwKNmHdkVxW2wZSoWFhQwaNCjm88pTK3QbuVYwlivBVTMwWhqpf2Wh2WJkDbNe2JqW84rqFbqdXHMJCUK2IgogA2l0Wti+xU6jM3d/PaIEBMF8cneFyVLWrCjixpn9ufe6Cm6c2Z81K4vMFiltiBIQBHMRBZBBBDeRazliwdVmYdGCMrEEBEFIC7m7smQhgSZyQfibyOUyogQEwRxEAWQQ0kROEITuRBRABuFvIucoMCgqMXAUGHnXRE4QhO5DkpgzDGkiZ37B2KHWMvY1DeSYkq/oVeg0VRZBSCeiADIQaSJnnhJ4t2Ymj33wS2wWDx7Dxg3VdzBl6Iq0XtPR6pGKYsEUxAUkZCRmuIMOtZbx2Ae/xOUtotndE5e3iEfX38Oh1rK0XbOy5hDTlm1n4tu1TFu2ncodh9J2LUEIRxSAkLF0txLY1zQQmyV0upfN4mFf08C0XC94roDdbWDzasas35OxE8aE3EMUgJDRdKcSOKbkKzxGqAvGY9g4puSrtFwvMFcgCP9cAUHoDkQBCBlPdymBXoVObqi+A4e1hWL7YRzWFm6oviNtgWCZKyCYjUSchKyguwLDU4au4JR+67olC8g/V2DM+tDZwhIIFroLedKErKG7lECvQme3pX/KXAHBTMQFJGQVuVgs5iq0cbC8SBZ/odsRBSBkHZmsBBytHno3tKQ0kycd5xQEEBeQkCIanZZurV7OhIrhcCprDjHmg1B/ft3QXhl3TkHwIwpASJo1K4pYdHcZVjt43XD1fCeTZrSk5NwFzgZK6mppqhxMW1l5yGuZNGIyOKcfry+zZ8z6PezvV5Kwaycd5xSEYMQFJARIZBJZOmcYDFmxhFkzJ3DOdZcya+YEhqxcGvF9meASSkdOf7RzljpbxSUkpATZRghA4rv4wAyDtqPH/DMMknEFFTgbqL57Hra2VmhrBaB6wa3srZ7cwRIA811C6cjpj3ROi9fg9Pd2oa0WcQkJSSMWgJDULj5dMwxK6mox7KGLp2GzU1JXG/UzZloC/px+j1XhtlvwWFXSOf3h5/RaFWiwGUjrCCEliAUgJLWL988wWLSgDKvNt/inYoZBU+VgLO5QzWLxuGmqHJzUedNJOnL6g89pd3kZ/391WN1Hv1u/m0liAkIimPrUKKWeAs4H9mmtR5kpSz6T7C4+HTMM2srKWT//AaoX3Iphs2PxuFk//4GI7p9gzHYFuQptKV+M/ed0tHqkdYSQUsx2Af0vMMNkGfKeVEwiKy0zGD7SndIU0J0zZrNs+Tre/v0LLFu+jp0zZsf0uUwICqeDdLiZhPzG1CdHa/2eUqrKTBkEH5k6iaytrLzLXX8kzLYE0oW0jhBSidkWQJcopa5WSn2klProsPOA2eLkNMns4hNJIU03sVoC2VZpK60jhFSR8U+Q1noRsAjg2JNH6y7eLphAOgvBkqHRaWH7zHcY+uq3ojZ3k0pbIZ/JnO2akFUUOBvos2UTri+daSsES4Y1K4q4cWZ/7r2ugiuXv897O87r8J7unMh1qLWMzxpGpXW8pCDES8ZbAELmMWTFEqrvnodht6NcblarxTzLpYHXU1EIlgzBdQ3+1NZHNzzAKf3WhVgCgUpb71HDMh1plWYMmheEWDB1m6aUehFYC5yolNqllJprpjxC1wRX6DqOHMbuauX3bVfSl/rAe1JRCJYMgbqGIPpZ63GNuTNkd98dE7nMGDQvCLFidhbQJWZeX4ifQIVue3sGAFVg43jjC5oc5SkrBIuFRqeFpm1OqtgBIwYGsoXC6xrm8CKLm+di/ZUNq8fNpnFl1A3t1S0TufyD5l1B+tA/aL67hs4IQjTEBSTERaQKXTturnmpmNrm+m5LIV2zooiDd67kce9VuLFTZHfz4YKF7JwxO6Q6uZ+1nsXNcymmBZp8nx2zwRnoqJnutMruHjQvCPEgQWAhLvwVup6CQlwlPfEUFLJs7sO09i5PeSFYNBqdFl5d4OFx71UU00IvGnG4W6hecCsFzgbAV9fwm+V7mP/Tj7GWhC7Ahs2OuvDhQJpoOtMqu3vQvJB9ab1mIhaAEDc7Z8zm8xOnsO2Fep7+80gOPlOBd3H60z/9Q2eaGhXHWr/EhcO3s2/HY/E1i/O7gkrLDAomD8B6b/SeQt0xU6A7B83nO5LWGx+iAIS4WbOiiMcXVOJ2KUCBy3d80YIyRlW3pcUKeOvlYp59oAyrXeP1QB+3xuG/cDs2o2OzuFh7CqW7crg7B83nKzJAJ37kWxHiwp9i6XZ19B6mK/1z9cvFPHVvGaDalQ402I/hKvUkT3ivPBoDmL8wYtuInTNms7d6ctTJYn5ytX1EvtBdab25hHwrQlxEah3tJx3pn41OC08v7A2ETsay2TVDFk7n93zYIQsoErH2FEq3EnC0eqSPT5rojrTeXEOeQCEuIrWOBo29QKcl/bO+zorNAZ4O7aoVVSPclJb1oo3RKb2mXwmkerFOhX9aFEh0uiOtN9eQb0aIi/ABMB43XHBlI1Mvak6L77+i0ovRIZlDc/m81Csbf5C5otLLkFE/ovrOG1MWTEyFf1oCnF0j3VLjQ74dIW66s3V0sMKxWH0K5wfzDjL1ouaUXie4oV2Zq54v9U9SGkxM1j8tAc7YScdQnlxFviUhIUrLjG7r9ZMKhePf3Q8u3kf/5h0hweDw3kEj2UkLdhxBKabJBhOT9U9LgFNIB/LkCFlBMgrHv7ufw4vc0nYVFNixaxfvznuQ+otmdQhs11CFndCgQ7LBxIB/et1uUAp0fP7pbA9wSuwiM5FKYCGn8e/uS9sa+F2br3K4uK0Ru6uVSffcyicvN3UIbO+ngqutT4ZUO6ckmKhBKV82k//fWMnmcZCVNYeYtmw7E9+uZdqy7VTuOGS2SEI7mf/0CEIS+Hf3VW01HSqH3dh5d+FBqqb2DAlsez1QNv9cllWvC6kdSDQ99FBrGYedfZn5wetYg1w48frwszHAKbGLzEZ+A0JKCM6gSSY2UOBsoPe2zSjAOWJUh9z9eK/j393XUNWhctiOm2ZbD2zvf8I5kwcwanlonKGNo7UDBc4GCmYuRL96c1wLl38WwOms5zLvu/SiMfBaIj78bAtwSuwis5HfgBA34YtwqkZCDlmxhAl33ozF68v7NOx21i14mJ0zZgOJjZ48mkVUzjXGkzzu9lUO23HzJHNZ03Kar030vb4WEf5rhcvlH4BjcR9tJ90VwbMAtjEy5XGFbCDbYxe5jtI6e8bsHnvyaH3P88vNFiOvCV+Ev3frQZ57oLcvg6YdR4HBb5bvCdmhd7VzL3A2MOu8amyu0BJjj6OQZSvWUU8FN87s3+V1/OcKb/vgv379hkN8+N9Omqw9WNt6WmgzuYIC3ntoMQeDLI8CZwOzZk7AFjT/wFNQyLLl67A9OafT7+qzhlHc+fZTNLt7AvBdXuQp5mK1urHiZVN1/4BLx22zYPcYWePaiYfKHYc6FGdJ/UJ8zHpha1KfV0pt0FqPDz+eW0+akFYijVp8dqGvQVtwa4jwnkCx7NxL6mrRVmuHa7a6rGx/ZR9tE/t3aEERqfdQ+G7dv6v3ZxENH1lMU3Ffttz/GW7sEKQArG1tnHnr1SjDCHwu0gAcw+brOnqgi7YR4bMA/sAl/M1yBk+ceT6Wshb67mli2rLtoDVWA7xWX2A4VQtkpmTeZGPsIl+QLCAhZiKNWrTaNJ5Q13pIT6BgpdHZ0PimysEob8c+QhYMHnlyNIXFukMLivDeQ+HjKm1trSEzAvzyPPtgbz73HtvRJQPYW5pDPhdpAE6kdtJ+gnvRR5oFcMmER/AM8J3PHxy1Gb5r27w6ZYPpMy3zJp0zF4TEEQUgxEykPkCG11eZ6ygwKCoxcBQYIT2BIisN3/Fg2srKWX/Xg3isNjSggTYcXMFinPYKWpsVV893Bq5jLzCYPbcx5ByB3XqwfO27dT9+efZTwRUsppkijlBCuCPU/7lIA3DC20n7lUCkRXfK0BU8OWsqd59zBU/OmhoYBh8IjkbAHyRNlODMG7vbSJlSEXKPTtWxUqoUqNBabw87Plpr/UlaJRMyjvA+QP75v5NmtHDa1NaIPv5ISiNa11D/oJk/fPcr3B4LmxjLfipweAwqKr0MH+lmVHUbq18uZsniUv7yTClLF5cGZOhqtx4uzx+4hNVM4zTbBpZZLgiJPwR/LpZ20p4rX2LMN8ZGTHeMNAsgUnDUT7JBUsm8EWIlqgWglLoY2Aa8opTaopQ6Lejl/023YJE42FbPvw5sMOPSQjv+UYu3/76e3yzfE/Dl+/zrHUdC+pVGNAshHMewMobffRrvFUyjqaQcR4HB5T8+SH2dNeA2WvpUKW5XR5dSLLv1cHkaC8oZfvdpbJh3F16HA3dxj4ifaysr58DIMVFbSpfU1WIU9Qg51tlOPriwy2PxWTweq4q7wCvS+EPJvBFiJWoWkFJqE3Ce1nq3Uup04BngNq31a0qpj7XWY7tRTp9MlUo7rivk9kkvcUKfcQD868AGtjas5aTyiYFjfjp7Tehe4s3f97//y612nnuwdyCAPHtuI395ppSWI0f3LkUlBrf/vp7hI32LbaQsoM7kGbXuVV/g2GbF4nKzYd4Cvvj29+K6v4jZQlbFW7OGd7qY+wO14VlAsQRwO+sOKpk3uUW6soA6UwD/0Fp/LejnAcBfgKeBH2qtT01KogRQlUpbrobL9ldw3tyN/OvABu5dMweP4cZmsXdQDJ29Jooh82l0WjqkftoLDNCETCSz2Q3+66V9DBwWv4+7szTPYOURi1IZsnLp0dGTrUcSXnRjafvsaPUwbdl2n8vJL3eYwsmULCAhecxIAz2slBru9/+3WwJnA0uAkUlJkwQ2rRjVXEzJwuls79OAp28rhgKP18v2165i7AHfH+f2Pg14KtwYePEYsLVhLSf0GZewYhClkVpiWVAjTR+z2eD8HzSydHEpWoPbpbBY4I5Lj0moAK2zNE+/XNFSS8MJjxV0VScQiVhbJ8Ti58+2qmGh++ns6biWsDl8WuvDSqkZwG1plSoKfTw2flo7kBGtRQCMai7GphUedEAx+BnVXIzN2I9HgU0bnLryJUqWLI2oNE4Qa6JbiXVBjRZAnnpRM6dPbeX2S/oBCleb7zFNZCh9V4Hj4NRSv5KoXnAre6snR1RcwaMn2xIYLxlrAFf8/EIq6CwNdAlwoVIqkK+nlOoH/A8wK81yRaTMawss/gAjWov4Re0QLttfwS9qh8T0ml9pWHSYNfHaVXi8rT6LwdvK9teuomTh9KOvGX5rws3WhrXAUcXwp60PcO+aOSEB6n8d2MDSzx6LGLTu7LVcJ5ZcfT+dBZBbmxU2R+gCGCm9NJxGp4XtW+yBgHJXgeNYUktTSawLezZ3BxUyh86elnHAfwGblFI3AV8DbgF+BXy/G2SLiRGtRSELf1ev+RXD5uJmRjUXZ4U1kUtWRiSXi9dqx/n+bjyTKzrs3qMNg4lkHXjc0NRoodFpCbwvONi7eV1BxIrkSGmefheVu7gkooWwp3gotVvsXQa1Yx0yH+yvj3WurVTYCsnSZS+g9sX/YaAOmKC13tUdgkXiuPIi/dC5VWk7/7bClg6KobPXthW28PPBO/Eon9LwWxov92ng+b71GAos2he0/nZ7bOLlPg08X3EAAy8WrHznpFuZffz1URVDZwoDoiuHWHzsZhAp6NpMESNKvmSvpyIuP/6alUWBmgRXKyiLwl6gA4s7msCC73GD4QGvt+teQuEuqu0XzGH4kpd8wV2Pm2dn/4Zrl86NqyldZ0ogUtBXFnYhmG4PAiulegP3A9XADGAmsEIpdZPW+q9JSZOhZJI14Q9ab21YG+R+OhrMhujWxJAVSzAW/5h3q+CsGrBc+VDAx262NeF3uVQvuBWv1Y632cMVLKa2qR8Qnx/fbx3UbLPx4C19cbcpPG5fPODxu8pAgTuobxFEdhkFXyuSz3/4kpdY+cJy7M1N7CkeyrWXjgzphxSLzNEsgWhB37dmDedgeeRnMdWkM1tIMpEym85+IxuB3wI/0lp7gDeVUmOA3yqldmitL+kOATOdeBRDZ6+FKw2/Yji1sIXXBhsdFAYQ0QVVXdcLY8PnnHs5uKzgmAhvPHkLBdWT+YeuSciaSDV+l4vz/d0s+NXYwOIPkRflzigtMygp1djs4A7KFrJYCEth6EikiuRoWUH25iYOjBxD7RZ7TE3pIuFvGRGsCLqjavdQaxn7mgZyTMlXHSqSY0k5TZR0nltIDZ09YVPC3T1a603AJKXUVWmVKgdIlTXRmTKJZGkUN7l5d5jCZdV4LeDS8O5gN1Pun83245ujps1+PPe+uGMTm3ZsZGPtek4dXM2YofGVhdRTQU3fSva4Q91T0dpEdFZIFrFHkUEHBWCzg1IGNvvRNhbh5+oqK6iwWONxhZ44mszR5CfIGugs6NvZwh0r/oE0NosHj2Hjhuo7Av2I0jmtSyaBZQdRfxOd+fq11k+kR5z8JprS6Ox4uHJoLvFw1jaNY7Jv8XcYcNaXmqZx9k5dU9tfuyqicthW2MK9/jiH9WgV9iuvb+aV1jlgdfHWXgcXbXmVi2aOCpyvM2siuD201gqr1cBRGH1R7qqddLQeRUDgWJm7nv+88hOOmdqf2uZjogZvg11Ufp+/PyvILwfKt6A5Cnz/dtbaIqr87UogMCw+LOi7as+sqAt3rAQPpHG166dH19/DKf3W0avQmVbrQ/oRZQfym8hywpWDq9BG30EDeOPZ3bw7THHWl5qiQQNwFtoY0WqLy5oA2FzcjEfpdsXgS489tu44ljTMgLNdYPGCdrHkzQ+YveFubD//Y6eB6007NvK7VZvxVpwDuyYCYC/QfPueFRzu/T59B03Al4DmI9IMgkg+92jZQqOq26h4ZSmzFt+C8Ywdy+L2uoORsylwNlC2bTMaQobARMoKCpGjHcPQXVYgdya/3xLwZ/P0craigV3F/XnsjegLd6zsaxqIzeIJnAPAZvGwr2kgvQqdaa0lkDqF7EAUQA5SN7QXjn4lTGly0zTOjjNoxxWPNQGRFcO+poHYdk7C63WAdoHhwLZzEvsK/8HxnVRobyts4eFBu/BOAc5wwNOrYddE1JC1vHjo2xgH3SzfGaow6uusMGQt9HsPas6GXRMDPvc9+sMQK8M/9CWYCuqZtfiWDoVctiOHGb/wzoC7x7DaWPeLRwLB8uCCLr8c4b5/uwNamzsPNET6XHDMwB8c7runKeAvP827mzm8xDP8e+AzwQt3rIQPpAHwGDaOKfkKIKr1kYodejrPLaQO+W3kKIm0AYg1NnGo5Ct07Rm+BbzqHag5G717DMeMvRPo3JrwWrw+37x2+T67ayLeAavRRkeF0TRvFYd7rMd1sc/VhNenNLz11RzusZ5HYohZTNht7RjUtVoZv3A+1iBfv9XrofquH3eo8PX77mMZSBOJaNXMg4v30WfLDpoqB0dsJf07fsRyzmc/FUDowh0r/oE0j66/J8SVFKxEkqkl6CrDR+oUMh/5jeQJyaTjhSuGkIVl76gOC0tX1oRbgzYcFOw6DW1t4aKST3klgsIoWTid3X0aUOWtaIsG7cJ67CpuGPIndr/13FEro5NeT3d97XGmBi30awfBX4e3MqXWyplfEHJ89XEGJV+sov84Xw+fcN/92Rc08c6SkpA4Q1eZP+HxCY8H7j/zKS679IZAncGWudf7WkkfORz4nLJ6OV5vo9laGHHhjkSkoPGUoSs4pd+6ToPJiWwWYs3wSaYfkaSQpp+sGgqf7kKwXCVd6XjxZKn433uoYhs1vfdSdbAfvepHBD4brQgvuNjOali585/HM+VgCxsqPPxseF1MRXjz33Rx7GeHWDsIpv0A2mxQ4IHVT8PEXb7Ff+oPfGmzVlsBt0/6A/3Vab5OpBXrA1aOo76ae17YR2uz6rICODxrqdFpYfXLxbz/ZBufu4d1GEaPJmQgjceqePXccdR6qmL6fjvL9kk1XXUiTcXCLSmkochQeCEh0pmOF2nSVSQiLU7jh66A8s2B98QSmzh7u8H3V/0TbVFMMjS9vtGHd461dBqzGHvQwdAv9qOAd6t8i7yhfP/+dRhM2AVvtx/3WkAbHrY2rMXqmuCLPVw8/aj76Y+raG0+LjB3IFqm0yuvb2bJ6o1YvzoLvXMiF8xt5PSprSx9qpTR7g24cIQoAMPmYOsPrmXk4kdDWkkX9jrC8WymK7rK9kk10TJ8DGcRloYipn36EdpCwgu3pJB2H6Z+m+2dRX8NWIEntdb3mSlPLmJ2Ol4qFqcRrUWMPmhn2qrtIYvC9988QGXYwJVw99O4egL3f3YNOLy+9FgbiuIhlaw9W9G3tA0b+9FaB4rtBhx8D++AGb7Fvz3TyTtgNUNfnQcjo2c6bdqx0ZceO8XlC5I/vZo//XYCrz3RCz14DZ9WruDjmhbOCUqytnjcbL/oMrZfdFlI5lGsnUTDs336Us8ItnDY2ZdeA6J/x4nWGUTK8NFeCz9992nW6zOwYUC7LIks3GY/s/mEad9me5fR/wamA7uAD5VSy7TWn5olUzYQr3ltdjpeV6mIsRLPohBsTTSXeAL3P3GXz+3z12OheHAlw42e7B8A/enB3bXFfOTQ9KsfzgDdSq9CJxeXfMqLQZlOF5d86pO5k0ynfxTaYPBRpeEPdHv6rYVLptNsdTHda2XV0w5O3VVISYGL9fMf4B+6xmdNDJjICe1B6KZ5q6h9dHJArtPb5QonONtnDi+ymLm4vXZK/naETdX9Iu7Ak3EZdczwgSuMJynUng7WTSILt9nPbD5hpjo9Hfhca/0FgFLqJWA2IAogCon4Rc1Ox+sqFTFWuloUIilG/w63eCxM+ni7L8Vyt8Y+pD91Rs+Qc+3ddhFL2xfERe0L4pyhKziuZjgfOWC8C8b3/nvg/dEynca7YEWQ0qDmbN8Hqt4JWBNeDbOH/ZifTJtO8WkD2Fu5nUfWfDdiI8BfDt2Lx3BD33pszy7npkHLmDJ0RWjMBLih+g5eXPefLDbm8vdBLbxT1cLZNXBahB14Kqyy4Ayfza7RvPZ/F1LsbsWBK+LvKB5rw+xnNp8w8xsdCAQ3Vd+Fr/FcCEqpq4GrASqK8/cBSMYvamY6XiypiLHQ2aIQSTG+qC8J2eH+7NSbmVq2MuL9d7Ygji90Mr61ozzRMp3GGx4u2TiLPzaNxPvF1ECxGzVng9eBRbVgRTPX8zFrnp7MlueK+WrSJoyvu9FhDf827VrnW/zbrQnPkLU8uvYeioet4v6wLrRThq7g6463+PALF+f5+0B54Y3nCezA/Uqj1+4BKbHK/Bk+PVv34zFs7KeCK1jMU8zFjZ0Sq88CSaSqWVJIu4eM/1a11ouAReDLAjJZHNNI1i9q5njAWFIRYyHSohBZMe5ltr4Jl3F0Qb9v48McN+sTKqind0NLyKKSqJsqWuB6Tu+/c17hTt5w9uCPdWOxWz14do/hoo2zKCtbziV/O8wZtauAVbg8Dv7ti9tZfaYDi92FzWLnpHKf0qhwTwbvr0OsCWUv4L0xs/A0bfV1iFUWNs6Yw4glS7GUtfB+lTcQ0HZpeG+IZlyJPbR1efm7GB9tgJ1n+wQetJa2Y1dxqGIbhCU2ddYi3U+wkv+z5XyO927jJyN/xonHfUg9FTy2LDFrQ0Zaph8zv92vgMFBPw9qPyZEINv9orFmDHVF+KIQSTF6sXCs+oI6hgSO2Sweyj73Mu3T7R1caKlyUwXTq9DJxaMWce5xfwoovgrqmfbSYWxBi2wBLpbtup9jn3uWr/14I5NPPj2QUTTu2LH8z3+twlN5tAq6Dc37v5uJ5fsPtd+XT2E0zbueJqDv8odxtDwU6APV9xu34Jx5Mxs/ewzP1gfalYbi9KtWs/GXU2DIWlwXT8ewu7jfYucXX/brct6F/7VgxRBJybuwsa8hNTEgIT2YqQA+BI5XSg3Dt/DPAS41UZ6MRvyikYmkGK0YfKGPDTnW2+tkxqcfRXShpcpNFYlgxVfc4MaDDRuhvYMMLBxbewzfHvmjkNqC0jKDa68cweN3TcQd6ECq8H5xBrZn3mLWnX9hzKAJISmopTNv5uc7TuFftas5YfBUSod+HYCTyidis9jxGD6lMWPieH64fA8vb1nO6gOugPtp44w5DD7+eoAwpWEJvPavAxu4971vdVAMvQqd7O5dx6ogxRCiXAethap3cO2cmJRyTTX5XHBm2t1qrT1KqeuBN/ClgT6ltd5iljypJF0PVKr8otn+wIfL31Ex9uMSHglZ0H9y8s/Q2wikJ0KoCy0VbqquAp17bBUYxl4IUwBWvEwf9zwDnnw3MDPA36juW72g392n8ssFJ9LWcrTvkH3vRMZZj2N4n7A+E0DV0K9T1b7w+/maquK+Ab9gg30nxw+a2q40DCaffDrvrTmqGE5xnEyfLZtoqhzcQWn4XVNbG9bisVgiK4ZAauxhnzWBL0D9612z8Fw+0xcI14rduwbSK4o10Z3ke8GZqX/9WuvlwHIzZUg16X6gkvWLZvsDH03+cMU4hdAFvYJ61Kedu9CScVPFklZZ66niOst/sMi4Bge+hduFg9dPqmbcce8G3jdkxRIm3HVLoFHdWVYb23ia54MM5OA+RNFmJfiPT9j6Mmc/eGug9cT6+Seyc4bPajihzzhun/QSWxvWcs6/3Mydc03gff3mPwDtrwUXu3WqGIKm1/kVwzjg/E/+m6U1PkvDwBp4rfbRyVHdTBBdOaRCaUjBWRYEgbOJTH+gYpUvUy2EruQPlzV4QXeRPhdarGmVx5R8xUtqDss5n7N4m37s5T01mZ+M+BGO9vf4RlLe2qFR3VO2ufzVMZUD1goM42gfomizEvzH+9nquaVpHjZCu6EGN707oc84vqaqmPX9CR26pu5dvo4Tjg+d5xCsNGJRDABjBk1g+c6Or22cMSeimwnCRp4a+wPKIZ7YRDDhr0nBmSiAlJLpD1Qs8mWyhdBZC4LPHCd26bqJ14UWa+56rFlE/ljD3rWn8Li+Djd2ilQzn+ytCHzHJXW1aIu1wzUMZWGoUcMBa0VgtHG0WQNDT3QHjvdr2xmh9YSdkrrakK6n0UZhhr/Pzwl9xnUY9BNNMXT2WmdKI8SisBy1GuKNTUDkgPbo9vjR2kHwThW+uond8SdWpGJym1mYvyrlEJmeqRNLMVX4DvuUdfv4ovdACnsdiXjO7rQWorUguOq9v3DQWhZTjnmsLrR4KmUH22o4xbuJzxjRZfvmaf3+zDTLr7F5DaAFjFArRr96M8ro2GLacBt8zrG0eXwDaRYtKOOWhxoizhr4fPPRucU1VHUozgoecemnq1GYsRJJMXT2WmdKI5pyiDc2AWEBbTSbi5sZ0VrOM9/ow/WnNgTqJh7b2Ie+7c9HLNaEe/tUXl31VLc04UsHlq7fIsSKPyDpsSrcdgseq8qoTJ2u5AvssINoNkp4ZOVDvLfjvA7nq6w5xLRl25n4di3Tlm2ncsehbpbfwhX6SeqMITS7e+LyFvHo+ns41FqW1HWCXTpdnbey5hAXvrGB1WoaOxjK9y3/g8PaEjWLyPcdhx7zW2H+e1w//0G8djsa32bfY7VxbcHigHIB30IPkWcUHDfKHTjuL85qpoi24p54CgpZP/8B6qlg+xY7jU6fMP5RmJ6CAtxFxXgKCgKjMNPNCX3GMfv466Mqh++cdGvIvIdox/2KwYK1gzUR8pq1kOHfeoKmeat484dX0ma34LVAm93Kmz+8kqZ5qwIWw/N96/n54J1sKzxqQQW/9tK4pbgGbAo8J7/eNYvnSptD3p/JZMbKlENkegVjZ/JF2mHbcfOZMYINYT5ts+IdkVoQELQIpiLHPFaXTvB3YGvfZT/JVfzbub+NajHFYiX6R1L6x1XW9h/NHy4dGfIZrweqRngizkIeOCz0+Gue7zLsx6dy9kmf0VQ5mLfXDWLRzAgzlrUOuJcIFdE0olkUqXVBOQJtOGKJTQRbE1iO9nti0Fo8l8/kZVsLS5OITXQnmbU65QiZXsEYTT7/DvuUdftoNkqw4+YKfDvPYsvhkAXQzHhHeAuCYJIt4ILY+xdF+g6wavp76jlI5D/kWOs52srK2TPxLAAc+BbpV+/ycKylhi+MKi6cb6O0zIg6C7nj8RIOMCZq3GD8iXVU3z0vZCZBeLA4G0iVC6qz2ETwa0aEfk9aEaI0Ost0ijWgPSu1X1OAzF2l8hSzM3Dqhvbii94DeWTlQ3xmRPdpZ0K8I10FXLGeN9HvoCsr0d8G2l8TADD7yEs8qOfhxYZduVnPA+xkNkDEWcjRjkebUezZ/FVcQeBcIhlrwvPZ2fy5vhpriYG7bgpY7BjEnukUOTbRMdPpzNq1TBw8kVQjCiCDyJQMnMJeR5g04TU2rL+HYsvhiAtgplQmp6rPUCLnjfYd1FPBvobO5YnHSvzk5SZ+fu88HLQE3F2x7M4j1QdEm1FsGzUwJUHgdBKt3iFdxGRNHA/TJ+5pl2sYe3Ti1oTNavfFJvqM65Dp9E7NO2lRADISMkPoasyeGcSS3ma2xZIJBH8HiXS+jEbTvFU0Oi08PaOW5e5z6c3RIHtbcU/eefwFDowcE/Gz0eoDANasLOoQN5g0o4UhK5dSveBW31Qyj5v18x9g54zZCcmeajq7n2wg2vS4aK+FDxx654d/TUoBRBsJKQogQ+jd0MLEt2uxu4/ubNx2C2vPGczB8u4NDAmxEa4gD7WWceWy1bi8R39fDmsLT86ampBl0jRvFdu32Fl0LWxrCp0j7HYU8ucV6yJaAI1Oi2+ecdvRdCNHgcFvlu8J7Jyj7aYLnA0hU8kygVjuJxcJVgzzpyaniGUmcIaTCT71fCTRIp5IdQIDetSmtPNlycLpVFy5mr2e/iF99u24WTMveopmJD+/xQofv1/A2MltgdhApMWzraw8YxZ+P9HiFvV11pxWAJ25oFKF1AFkCJleQ5AqHK0eeje04Gj1xPTeit1H6Lv7SEzvj5d3a2Zy5bLV3Pn2U1y5bHXEWodIRKsTKLQ1dZo9dKi1jM8aRsVVp1BaZnD1fCevFXyXEcVfMsP2Jjf8+yds//oFUT8Tyc/f2qx4+ldl3DizP2tWZpdFGS1u4e+FJCRObq0uWU6m1xAkSzxB7sqaQ4xdtxtLu1FkWODjCQNSFhRPZixitDqBVk9J1OyhZGbw+lM6V79SzJLFp/H3P8FzL0T3g/uVxqIFZVisvsUfFK1NviI/f7uI1mbVbQHVZAi+n+C4RabLnQ3k1gqTA2RiDUEqAr3xFI45Wj2MWb8ba3B6vQFjU1holsyw+s7qBI4v39wheyjZGbyNTgs122wsWVyKu82COyh/f1R1W8SFcNKMFoae6GbNG4Usf66UtuajFd5aw+2X9MPm0FkTUI1W7yAkR2atNEJGELzg993TlJLU1HgKx4qb3KAU4eWoGlJWaJbMFLCu6gTC20ono2w++tHDPLZxIcoC7rbQNh2d+cHXrCji1QUehln+Sc/WYbRxTOA133AZFRgy05kiySSixS2ExBEFIIQQ4qbxGih8u+9k2z3EE+RuLrH7tqlhKP9rKSDZIrJ46g8SVTZHLYfIobpofvBGp4WDd67kM+9VuHDgwMUVLGZp8XfxuMFiAVeQMrHaoGabnQr2UcUOGDGwy0Bwd+fkC+lBFIAQIJKbJnwZTrTdQzyFY65CG5smDGDs2rAYQIqD4skWkcU6QCZRZRPJcgBNQZFGG9H94E3bnDzuvYpiWgKpo08xlzuvP5X+1b2449JjQt7valXsvOFNfm5c5WtRbXfz4YKFUWsAUpWTL0rEfEQBCAEi9rYJo6vU1M7iBfEEuf3v7eVsRQONZYVpiY2kalh9VySibCJZDnaH5uYHGqga4Y66aFaxAzd2CK4bwM6kodtpGzY6EFDV2ucOKvPW8wRXtSsLX7VxtErjaL2E4nUhZXthV64gaaBCgEhuGkOBN8bU1FjaQ7sKbRwsL4ppMXcV2qgf0IP9A3pkXGA8EXoVOjm+fHPMCsdvOTisLRSVGDgKDK65y8noiR0X20an5Wh75xEDKbKH5k0W2d0wYiDgC6je88I+fE41RRU7cAVmkvnwWHx9gMIJ5OQH4Y9FxEqwEmk5YsHVZmHRgrJAa+p0U+BsoM+WTRQ4G7rleplM9v9VCRFJJHMnmpsmll17po/DzFb8lsOOC18LuErCXScdd9MF9FuwkOoFt+Kx2LEZbj6cvzBkN9/arLA5NG6Xijg0xma42VM8lNot9ph6CcWTk29mYdeQFUuovnte0HzkzGl3YQbyl5mDJNNULpqbpqtFPNPHYWYzvQqdDB/pptFp4bUne7BkcSm29sX+8h8f5NkHe3d0ySz/FnuXT47a1iF4IfcPjfFXGhfZ3Tw/+9dce+nIDi6aVOTkm1XY5Zu3PK/j3OMsa3mdSuQvM8dIxU48kVoEaWWRXtasKOLxBWVHUzjbF/unF/bG5iDybnpk9LYO4Qv5q57vUnLpafzb+H9xuP9grrl0JO4ofv5kc/LNKuyKd+5xPiAKIMcwayfudx+NXb8Hjc+7nIutLMzgUGsZi+4sxh0hHdRnCYTWB8S6m+64kBfSxmhef6JHlzUHyebkm1HYlaq5x7mEBIFzDFN34hr83WUzrctsPD2IMg1/OmgkDC9cPs+Jo8AIBIrj2U2XlhkMH3k0o6jRaWHJ4lJ8KvwoHg8UFuuQOcLJEn7taIQEuJPg6NzjQlwlR+cj5+vuH8QCyDnMGtQScD0Z4K8eyJQgcKYM2kmUSOmgoLEX6IBv/vSvtya9m250Wvj4/QKsdnCHxIQ1p05p4Y5Lj0k4bTPRnP9Up4v65y1nWstrsxAFkIOY0VQuU4PAuZCdFFxIZi0swOOGC65sZOpFzSlzyfgXWouNQNM4Pza7ZuN7RVFjArGeO95FPFU1B+FkYstrs8iOv4AsxqyJWd3dVC5Tg8CZqpjiJVI6aKqItNCCprBYY3hh9txG/vJMaSDwDLGnbSaziOfrHIBwGp0WPvwQqqqgoiK1586ev4AsJNtdD/GQKTOCw8lUxZQI/nTQVBNpoXUUaL57/UEmnuvLmFm6uDTkM7EGmuNdxIMnklVUVuT9HAC/9VRUCC4XLF4Ml1ySuvNLEDhNBLse7G4Dm1czZv2erAxCxkrd0F68NWs4a88ZzFuzhmeEssuXQTvJECkv39WmeO6hMjavLwikbSYSaI4n53/IiiXMmjmBc667lFkzJzBq/WsJXzcXCLaeDh2ClhaYOxfq61N3DZkJnCZkxm9mkUvD65vmrer09UQCrm+9UsxT95QRnv1jLzB4tH32bsKB3ChD6IMpcDYwa+YEX5FWO56CQpYtX0c9FSHXNaOJnBmzkrdvsXPvdRW0HDm6Ty8thbfegtNOi+9cMhO4m8kl10MukImDdtJBogHXYSPcOAo1rtZQBWCxHHXXJBpojiXnv9MirZHlgc+Y0UTOrPYRkawnt9sXC0gV4gJKE+J6ELqbZJqsVVR60RHWdsNIjc+9q5z/WIq0zGgiF9w+wnHkMLa2VqoX3NotjeSCXW+lpVBU5IsBpDIQLKtRHMTrRsj1Gb/5Sqa6k2IJuEZzn5SW+TqN/u7nZXi9PivAZodrusnn7i/Sql5wK4bNjsXj7lCkZUZWkNntI/zW06k9KyULyEwSzejJF9dDvpAJmV0lC6dHjANEchmUues5uXEDOAfy9rpBnbpP/ItNzTYboDqdOZAOuirSMqOJXCa0jygtMzhtbHrOLS6gGMikjJ5sbmmQ7WTScxCJ8Gyd71mf50s9jPN/OodZMyfgvPONLt0npWUGoye6Is4c6A7ayso5MHJMxN11MtlIyciTy+0jZGsaA5lSTJQJu898JlOeg87w7+Kbtjm58ZYrfVk17RvYRVzJG0xnPz4/QjYWVZnRRC6X20dkxlOb4WRCRk8utDTIdjLhOYiF0jKDqtIvO/iu3dipoiagALK1qCrZtheJkKvtI8QFFAOZkNET2H0G4d99Ct1DJjwHsRLJd11kd1PnGJKXRVVCZEx5cpVS3wHuAk4CTtdaf2SGHPFgdkZPtuw+cx2znwM/0QLBfiJl1Xw4fyF3Vnupr6vv1iIqIXMxa+uyGbgQeNyk60ekq/Q+MzN6MrXXTj6SLZldkXzXpXS/+0TIXEx5irXWWwGUUl29tdvIhgBrpuw+hewh13zXZrRkyGUyPgaglLpaKfWRUuqjxjSl22V6el8wrkIbB8uLZPEXKFk43WwRupXwZnFDVi41W6SsJ20KQCn1llJqc4T/4mqiobVepLUer7UeX5qmRU8CrIKQ2ZjZkiGXSds2Ums9LV3nTjUSYBWEzMbslgy5Ssa7gLqDbErvE4R8JBNaMuQipigApdS3lFK7gInA60qpN8yQI5hMHGYiCF1hZhygwNlAny2busUNk+stGczCrCyg14DXzLh2Z2RLep8gmI0ZPfJzuSWDWchqJwhCXAQHZP0++eoFt7K3enLaF+VcS2s1G4kBZBHSCVTIBAIB2SD8AVkhuxALIEuItVAtU4eVCOmjq7YQqSbegKwZM3yF2JAVIguItRNoNlQzC9lPLNO7/Jgxw1eIHVEAWUAsfeilXbTQncQSkA2e4esf47hoQRmjqs0ZNiN0RFaGLCCWQrVsGFYi5BZdBWTNmOEbC+KSOooEgbOAWArVpJo5v8nEvkBmzPDtijUrirhxZn/uva6CG2f2Z83KItNkyQRka5gldNUJVNpFC5mGf4bvogVlWG2+xd/MITTRXFJDT3TT2qzy0iKQ1SGL6KpQTdpFC5mGGTN8oxHJJQVw25xjsBfkZ5BaXEA5hrSLzl8y0Q0EPktg+Ei36bvrSC4pV5vC47bQcsSCq83CogVlNDrzZ1nMnzvtAimyEoTcprTM4Hu3HsTu0BQWG9gdGntBaNzMH6TOF2SbiOTPC0I+sGZFEc890BurXeNxwcX/cZCXfx/6d252kLq7yXsLIJumgQlCrtHotLB9iz3tbpfgAHBrkwWP28LLv+/F9249iKPAoKjEwFFgmBqkNoO8twAkf17IJbq7LUQydGeVcLSahGEj3Pxm+Z6MCFKbQd6vcJI/LwjdT3dUCQcXfHVWk1BaZuTdwu8n711AMg1MELqfwI48iFQGYMMLvjavL+Dq+c68dvdEQlY5JH9eELqbdFYJR7MufrN8T167eyKR9xaAH8mfF4Tuw18lnI4deWfWRabUJGQKstoJQo6RLYHgdFUJZ2IPokxFLABBEEwjHTvydFoXuYZYAIIg5ByZ1IMokxELQBBykEztC9SdiL+/a0QBCIIg5CmiAARBEPIUUQCCIAhport6HSVKZkolCELSSBzAXLJh/KQoAEEQhBQTXI2cycNmMksaQRCEHCDdvY5ShSgAQRCEFJMt1ch5rwBkFKSQy0gcwByypRo5ryuBZRSkIAjpIhuqkfNWAQSPgvRPAxuzfg/7+5VIR1BBEFJCpg+byVsXUGAUZBD+UZCCIAj5QN4qABkFKeQLEgcQopF3CsAf9AVkFKQgCHlNXq12kYK+b80aLqMgBUHIS/LGAggO+trdBjavZsz6PQAyClLIecQNJERCaa27fleGoJSqB3Yk8tkeUHw8nGABaz1QARjg/Qz+dQSaUypo99IX2G+2ECkkl+4nl+4Fcut+8u1ehmqtK8IPZpUCSBVKqY+01uPNliMV5NK9QG7dTy7dC+TW/ci9+MgbF5AgCIIQiigAQRCEPCVfFcAiswVIIbl0L5Bb95NL9wK5dT9yL+RpDEAQBEHIXwtAEAQh7xEFIAiCkKfkrQJQSv1CKfWJUmqTUupNpVSl2TIlilJqoVJqW/v9vKaU6m22TMmglPqOUmqLUspQSmVlqp5SaoZS6p9Kqc+VUj8zW55kUEo9pZTap5TabLYsyaKUGqyUelsp9Wn7M3aT2TIlilKqUCn1gVLq7+33siDuc+RrDEApVaq1bmz//xuBk7XW15osVkIopb4B/FVr7VFK3Q+gtf6pyWIljFLqJMAAHgdu1Vp/ZLJIcaGUsgL/AqYDu4APgUu01p+aKliCKKWmAEeAZ7TWo8yWJxmUUgOAAVrrjUqpnsAG4IJs/N0opRRQorU+opSyA+8DN2mt18V6jry1APyLfzslQNZqQq31m1pr/0izdcAgM+VJFq31Vq31P82WIwlOBz7XWn+htXYBLwGzTZYpYbTW7wEHzJYjFWitd2utN7b//2FgKzDQXKkSQ/s40v6jvf2/uNaxvFUAAEqpe5RStcBlwJ1my5MirgBWmC1EnjMQqA36eRdZusjkMkqpKmAssN5kURJGKWVVSm0C9gGrtNZx3UtOKwCl1FtKqc0R/psNoLW+Q2s9GHgeuN5caTunq3tpf88dgAff/WQ0sdyPIKQLpVQP4BXgP8O8AVmF1tqrtR6Dz+o/XSkVl4sup1tgaq2nxfjW54HlwPw0ipMUXd2LUuqHwPnAVJ0FgZ04fjfZyFfA4KCfB7UfEzKAdn/5K8DzWutXzZYnFWitDyql3gZmADEH63PaAugMpdTxQT/OBraZJUuyKKVmAD8BZmmts7mzaa7wIXC8UmqYUsoBzAGWmSyTQCBwuhjYqrV+yGx5kkEpVeHP+FNKFeFLOohrHcvnLKBXgBPxZZvsAK7VWmflLk0p9TlQADS0H1qXrRlNAEqpbwGP4uvafRDYpLU+11Sh4kQpNRN4BLACT2mt7zFXosRRSr0InI2v7fBeYL7WerGpQiWIUmoy8DfgH/j+9gFu11ovN0+qxFBKjQaexveMWYA/aq3vjusc+aoABEEQ8p28dQEJgiDkO6IABEEQ8hRRAIIgCHmKKABBEIQ8RRSAIAhCniIKQBDioL2b5JdKqT7tP5e1/1yllFqplDqolPqL2XIKQiyIAhCEONBa1wK/A+5rP3QfsEhrXQMsBC43STRBiBtRAIIQPw8DE5RS/wlMBh4A0FqvBg6bKJcgxEVO9wIShHSgtXYrpeYBK4FvaK3dZsskCIkgFoAgJMZ5wG4gqwekCPmNKABBiBOl1Bh8jbcmADe3T5kShKxDFIAgxEF7N8nf4esjvxNf4PcBc6UShMQQBSAI8XEVsFNrvar9598CJymlzlJK/Q34EzBVKbVLKZVVHUyF/EO6gQqCIOQpYgEIgiDkKaIABEEQ8hRRAIIgCHmKKABBEIQ8RRSAIAhCniIKQBAEIU8RBSAIgpCn/H/azeb1KuLThwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "def draw_line(ax, v, p0, rect, N=50, label=None, color=\"black\"):\n", + " x1, x2, y1, y2 = rect\n", + " v = v / numpy.linalg.norm(v) * (x2 - x1)\n", + " points = [p0 + v * ((i * 2. / N - 2) + (x1 - p0[0]) / v[0]) for i in range(0, N * 4 + 1)]\n", + " arr = numpy.vstack(points)\n", + " arr = arr[arr[:, 0] >= x1]\n", + " arr = arr[arr[:, 0] <= x2]\n", + " arr = arr[arr[:, 1] >= y1]\n", + " arr = arr[arr[:, 1] <= y2]\n", + " ax.plot(arr[:, 0], arr[:, 1], '.', label=label, color=color)\n", + "\n", + "def zones(ax, model, X):\n", + " r = (X[:, 0].min(), X[:, 0].max(), X[:, 1].min(), X[:, 1].max())\n", + " h = .02 # step size in the mesh\n", + " xx, yy = numpy.meshgrid(numpy.arange(r[0], r[1], h), numpy.arange(r[2], r[3], h))\n", + " Z = model.predict(numpy.c_[xx.ravel(), yy.ravel()])\n", + " Z = Z.reshape(xx.shape)\n", + " return ax.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)\n", + "\n", + "fig, ax = plt.subplots(1, 1)\n", + "zones(ax, model, X)\n", + "df[df.y == 0].plot.scatter(x=\"X1\", y=\"X2\", color=\"blue\", label=\"y=0\", ax=ax)\n", + "df[df.y == 1].plot.scatter(x=\"X1\", y=\"X2\", color=\"red\", label=\"y=1\", ax=ax);\n", + "rect = (df.X1.min(), df.X1.max(), df.X2.min(), df.X2.max())\n", + "draw_line(ax, model.centers_[1] - model.centers_[0], model.centers_[0],\n", + " rect, N=100, label=\"hyperplan\", color=\"green\")\n", + "ax.legend();" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conversion to ONNX = second implementation\n", + "\n", + "The conversion fails as expected because there is no registered converter for this new model." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MissingShapeCalculator\n", + "---\n", + "Unable to find a shape calculator for type ''.\n", + "It usually means the pipeline being converted contains a\n", + "transformer or a predictor with no corresponding converter\n", + "implemented in sklearn-onnx. If the converted is implemented\n", + "in another library, you need to register\n", + "the converted so that it can be used by sklearn-onnx (function\n", + "update_registered_converter). If the model is not yet covered\n", + "by sklearn-onnx, you may raise an issue to\n", + "https://github.com/onnx/sklearn-onnx/issues\n", + "to get the converter implemented or even contribute to the\n", + "project. If the model is a custom model, a new converter must\n", + "be implemented. Examples can be found in the gallery.\n", + "\n" + ] + } + ], + "source": [ + "from skl2onnx import to_onnx\n", + "one_row = X_train[:1].astype(numpy.float32)\n", + "try:\n", + " to_onnx(model, one_row)\n", + "except Exception as e:\n", + " print(e.__class__.__name__)\n", + " print(\"---\")\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Writing a converter means implementing the prediction methods with ONNX operators. That's very similar to learning a new mathematical language even if this language is very close to *numpy*. Instead of having a second implementation of the predictions, why not having a single one based on ONNX? That way the conversion to ONNX would be obvious. Well do you know ONNX operators? Not really... Why not using then numpy functions implemented with ONNX operators? Ok! But how?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A single implementation with ONNX operators\n", + "\n", + "A classifier needs two pethods, `predict` and `predict_proba` and one graph is going to produce both of them. The user need to implement the function producing this graph, a decorator adds the two methods based on this graph." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from mlprodict.npy import onnxsklearn_class\n", + "\n", + "@onnxsklearn_class('onnx_graph')\n", + "class TwoLogisticRegressionOnnx(ClassifierMixin, BaseEstimator):\n", + " \n", + " def __init__(self):\n", + " ClassifierMixin.__init__(self)\n", + " BaseEstimator.__init__(self)\n", + " \n", + " def fit(self, X, y, sample_weights=None):\n", + " if sample_weights is not None:\n", + " raise NotImplementedError(\"weighted sample not implemented in this example.\")\n", + " \n", + " # Barycenters\n", + " self.weights_ = numpy.array([(y==0).sum(), (y==1).sum()])\n", + " p1 = X[y==0].sum(axis=0) / self.weights_[0]\n", + " p2 = X[y==1].sum(axis=0) / self.weights_[1]\n", + " self.centers_ = numpy.vstack([p1, p2])\n", + " \n", + " # A vector orthogonal\n", + " v = p2 - p1\n", + " v /= numpy.linalg.norm(v)\n", + " x = numpy.random.randn(X.shape[1])\n", + " x -= x.dot(v) * v\n", + " x /= numpy.linalg.norm(x)\n", + " self.hyperplan_ = x.reshape((-1, 1))\n", + " \n", + " # sign\n", + " sign = ((X - p1) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel()\n", + " \n", + " # Trains models\n", + " self.lr0_ = LogisticRegression().fit(X[sign == 0], y[sign == 0])\n", + " self.lr1_ = LogisticRegression().fit(X[sign == 1], y[sign == 1])\n", + "\n", + " return self\n", + " \n", + " def onnx_graph(self, X):\n", + " h = self.hyperplan_.astype(X.dtype)\n", + " c = self.centers_.astype(X.dtype)\n", + " \n", + " sign = ((X - c[0]) @ h >= 0).astype(x.dtype).reshape((-1, 1))\n", + "\n", + " prob0 = self.lr0_.predict_proba(X)\n", + " prob1 = self.lr1_.predict_proba(X)\n", + " prob = prob1 * sign - prob0 * (sign - numpy.array([1], dtype=X.dtype))\n", + " label = nxpy.argmax(prob, axis=1)\n", + " return label, prob\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/_doc/notebooks/numpy_api_onnx2.ipynb b/_doc/notebooks/numpy_api_onnx_ftr.ipynb similarity index 79% rename from _doc/notebooks/numpy_api_onnx2.ipynb rename to _doc/notebooks/numpy_api_onnx_ftr.ipynb index bdc3995f3..d8d65dfeb 100644 --- a/_doc/notebooks/numpy_api_onnx2.ipynb +++ b/_doc/notebooks/numpy_api_onnx_ftr.ipynb @@ -259,7 +259,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -296,16 +296,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -333,7 +333,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "4.43 \u00b5s \u00b1 311 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "4.2 \u00b5s \u00b1 541 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -350,7 +350,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "16 \u00b5s \u00b1 2.13 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "16.9 \u00b5s \u00b1 2.04 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -376,7 +376,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -435,7 +435,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -469,16 +469,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -507,7 +507,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "6.34 \u00b5s \u00b1 522 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "6.1 \u00b5s \u00b1 126 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -527,7 +527,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "17.8 \u00b5s \u00b1 722 ns per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + "22.4 \u00b5s \u00b1 4.14 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -551,7 +551,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "349 \u00b5s \u00b1 21.4 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "508 \u00b5s \u00b1 109 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -569,7 +569,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "367 \u00b5s \u00b1 50.1 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "568 \u00b5s \u00b1 83.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -601,9 +601,9 @@ { "data": { "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178631, 2.491644 , 1.3178631],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.1368184]], dtype=float32)" + "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", + " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", + " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" ] }, "execution_count": 20, @@ -656,9 +656,9 @@ { "data": { "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", + " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", + " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" ] }, "execution_count": 21, @@ -713,16 +713,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 22, @@ -794,11 +794,11 @@ "Onnx-Concat(Co_output0, Si_output0) -> Co_concat_result0\n", "+kr='Co_concat_result0': (2, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", "Onnx-Transpose(x) -> Tr_transposed0\n", - "+kr='Tr_transposed0': (4, 3) (dtype=float32 min=-1.8227919340133667 max=2.354652166366577)\n", + "+kr='Tr_transposed0': (4, 3) (dtype=float32 min=-2.9092655181884766 max=0.9116002321243286)\n", "Onnx-MatMul(Co_concat_result0, Tr_transposed0) -> Ma_Y0\n", - "+kr='Ma_Y0': (2, 4, 3) (dtype=float32 min=-2.7854056358337402 max=3.9340782165527344)\n", + "+kr='Ma_Y0': (2, 4, 3) (dtype=float32 min=-4.017709255218506 max=3.2469072341918945)\n", "Onnx-Pow(Ma_Y0, Po_Powcst) -> Po_Z0\n", - "+kr='Po_Z0': (2, 4, 3) (dtype=float32 min=0.0 max=15.476971626281738)\n", + "+kr='Po_Z0': (2, 4, 3) (dtype=float32 min=0.0 max=16.141986846923828)\n", "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst3\n", "+kr='Sl_Slicecst3': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst4\n", @@ -806,11 +806,11 @@ "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst5\n", "+kr='Sl_Slicecst5': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Slice(Po_Z0, Sl_Slicecst3, Sl_Slicecst4, Sl_Slicecst5) -> Sl_output0\n", - "+kr='Sl_output0': (1, 4, 3) (dtype=float32 min=0.7225952744483948 max=15.476971626281738)\n", + "+kr='Sl_output0': (1, 4, 3) (dtype=float32 min=0.045248985290527344 max=16.141986846923828)\n", "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst1\n", "+kr='Sq_Squeezecst1': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Squeeze(Sl_output0, Sq_Squeezecst1) -> Sq_squeezed0\n", - "+kr='Sq_squeezed0': (4, 3) (dtype=float32 min=0.7225952744483948 max=15.476971626281738)\n", + "+kr='Sq_squeezed0': (4, 3) (dtype=float32 min=0.045248985290527344 max=16.141986846923828)\n", "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst6\n", "+kr='Sl_Slicecst6': (1,) (dtype=int64 min=1 max=1)\n", "Onnx-Identity(Sl_Slicecst1) -> Sl_Slicecst7\n", @@ -818,25 +818,25 @@ "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst8\n", "+kr='Sl_Slicecst8': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Slice(Po_Z0, Sl_Slicecst6, Sl_Slicecst7, Sl_Slicecst8) -> Sl_output02\n", - "+kr='Sl_output02': (1, 4, 3) (dtype=float32 min=0.0 max=1.636295199394226)\n", + "+kr='Sl_output02': (1, 4, 3) (dtype=float32 min=0.0 max=10.542407035827637)\n", "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst2\n", "+kr='Sq_Squeezecst2': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Squeeze(Sl_output02, Sq_Squeezecst2) -> Sq_squeezed02\n", - "+kr='Sq_squeezed02': (4, 3) (dtype=float32 min=0.0 max=1.636295199394226)\n", + "+kr='Sq_squeezed02': (4, 3) (dtype=float32 min=0.0 max=10.542407035827637)\n", "Onnx-Add(Sq_squeezed0, Sq_squeezed02) -> Ad_C0\n", - "+kr='Ad_C0': (4, 3) (dtype=float32 min=0.8215643763542175 max=17.113265991210938)\n", + "+kr='Ad_C0': (4, 3) (dtype=float32 min=0.045248985290527344 max=16.141986846923828)\n", "Onnx-Sqrt(Ad_C0) -> Sq_Y0\n", - "+kr='Sq_Y0': (4, 3) (dtype=float32 min=0.9064018726348877 max=4.1368184089660645)\n", + "+kr='Sq_Y0': (4, 3) (dtype=float32 min=0.21271808445453644 max=4.017709255218506)\n", "Onnx-Transpose(Sq_Y0) -> y\n", - "+kr='y': (3, 4) (dtype=float32 min=0.9064018726348877 max=4.1368184089660645)\n" + "+kr='y': (3, 4) (dtype=float32 min=0.21271808445453644 max=4.017709255218506)\n" ] }, { "data": { "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", + " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", + " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" ] }, "execution_count": 23, @@ -857,7 +857,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "22.6 \u00b5s \u00b1 6.06 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + "29.6 \u00b5s \u00b1 8.88 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" ] } ], @@ -874,7 +874,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "269 \u00b5s \u00b1 4.46 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "299 \u00b5s \u00b1 17.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -898,7 +898,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "1.68 ms \u00b1 55.9 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "1.9 ms \u00b1 45.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -916,7 +916,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "3.35 ms \u00b1 76.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)\n" + "3.95 ms \u00b1 443 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)\n" ] } ], @@ -942,9 +942,9 @@ { "data": { "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", + " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", + " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" ] }, "execution_count": 28, @@ -972,7 +972,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "108 \u00b5s \u00b1 46.8 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n" + "73.1 \u00b5s \u00b1 23.5 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n" ] } ], @@ -996,7 +996,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "248 \u00b5s \u00b1 6.11 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "283 \u00b5s \u00b1 13 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -1065,9 +1065,9 @@ { "data": { "text/plain": [ - "array([[3.147964 , 1.5111852, 2.6032405, 1.5111852],\n", - " [2.7854056, 1.3178632, 2.491644 , 1.3178632],\n", - " [0.9064019, 4.1368184, 2.4568543, 4.136818 ]], dtype=float32)" + "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", + " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", + " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" ] }, "execution_count": 33, diff --git a/_doc/notebooks/onnx_profile_ort.ipynb b/_doc/notebooks/onnx_profile_ort.ipynb new file mode 100644 index 000000000..00dee2f89 --- /dev/null +++ b/_doc/notebooks/onnx_profile_ort.ipynb @@ -0,0 +1,1124 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Profiling KMeans with onnxruntime\n", + "\n", + "The notebook profiles the execution of an ONNX graph built from a *KMeans* model and executed with *onnxruntime*." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
run previous cell, wait for 2 seconds
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from jyquickhelper import add_notebook_menu\n", + "add_notebook_menu()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext mlprodict" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Builds a KMeans" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import make_classification\n", + "X, y = make_classification(100000)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "KMeans(max_iter=10)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.cluster import KMeans\n", + "km = KMeans(max_iter=10)\n", + "km.fit(X)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy\n", + "from mlprodict.onnx_conv import to_onnx\n", + "onx = to_onnx(km, X[:1].astype(numpy.float32))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%onnxview onx" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Json\n", + "\n", + "Another way to look into a model." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from mlprodict.onnxrt import OnnxInference\n", + "\n", + "oinf = OnnxInference(onx)\n", + "js = oinf.to_json()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import json\n", + "from io import StringIO\n", + "from jyquickhelper import JSONJS\n", + "JSONJS(json.load(StringIO(oinf.to_json())))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Profiling" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "from mlprodict.onnxrt import OnnxInference\n", + "\n", + "oinf = OnnxInference(onx, runtime=\"onnxruntime1\",\n", + " runtime_options={\"enable_profiling\": True})" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(0, 111):\n", + " oinf.run({\"X\": X.astype(numpy.float32)})" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
catpidtiddurtsphnameargs_op_nameargs_providerargs_graph_indexargs_parameter_sizeargs_exec_plan_indexargs_activation_sizeargs_output_size
0Session50478442418077221Xmodel_loading_arrayNaNNaNNaNNaNNaNNaNNaN
1Session5047844241804864850Xsession_initializationNaNNaNNaNNaNNaNNaNNaN
2Node504784424180428293XRe_ReduceSumSquare_fence_beforeReduceSumSquareNaNNaNNaNNaNNaNNaN
3Node504784424180470828305XRe_ReduceSumSquare_kernel_timeReduceSumSquareCPUExecutionProvider0008000000400000
4Node504784424180033104XRe_ReduceSumSquare_fence_afterReduceSumSquareNaNNaNNaNNaNNaNNaN
.............................................
2550Node50478442418002092121XAr_ArgMin_fence_beforeArgMinNaNNaNNaNNaNNaNNaN
2551Node50478442418032742092122XAr_ArgMin_kernel_timeArgMinCPUExecutionProvider5053200000800000
2552Node50478442418002095401XAr_ArgMin_fence_afterArgMinNaNNaNNaNNaNNaNNaN
2553Session504784424180113532084051XSequentialExecutor::ExecuteNaNNaNNaNNaNNaNNaNNaN
2554Session504784424180113672084044Xmodel_runNaNNaNNaNNaNNaNNaNNaN
\n", + "

2555 rows \u00d7 14 columns

\n", + "
" + ], + "text/plain": [ + " cat pid tid dur ts ph \\\n", + "0 Session 504784 424180 772 21 X \n", + "1 Session 504784 424180 4864 850 X \n", + "2 Node 504784 424180 4 28293 X \n", + "3 Node 504784 424180 4708 28305 X \n", + "4 Node 504784 424180 0 33104 X \n", + "... ... ... ... ... ... .. \n", + "2550 Node 504784 424180 0 2092121 X \n", + "2551 Node 504784 424180 3274 2092122 X \n", + "2552 Node 504784 424180 0 2095401 X \n", + "2553 Session 504784 424180 11353 2084051 X \n", + "2554 Session 504784 424180 11367 2084044 X \n", + "\n", + " name args_op_name args_provider \\\n", + "0 model_loading_array NaN NaN \n", + "1 session_initialization NaN NaN \n", + "2 Re_ReduceSumSquare_fence_before ReduceSumSquare NaN \n", + "3 Re_ReduceSumSquare_kernel_time ReduceSumSquare CPUExecutionProvider \n", + "4 Re_ReduceSumSquare_fence_after ReduceSumSquare NaN \n", + "... ... ... ... \n", + "2550 Ar_ArgMin_fence_before ArgMin NaN \n", + "2551 Ar_ArgMin_kernel_time ArgMin CPUExecutionProvider \n", + "2552 Ar_ArgMin_fence_after ArgMin NaN \n", + "2553 SequentialExecutor::Execute NaN NaN \n", + "2554 model_run NaN NaN \n", + "\n", + " args_graph_index args_parameter_size args_exec_plan_index \\\n", + "0 NaN NaN NaN \n", + "1 NaN NaN NaN \n", + "2 NaN NaN NaN \n", + "3 0 0 0 \n", + "4 NaN NaN NaN \n", + "... ... ... ... \n", + "2550 NaN NaN NaN \n", + "2551 5 0 5 \n", + "2552 NaN NaN NaN \n", + "2553 NaN NaN NaN \n", + "2554 NaN NaN NaN \n", + "\n", + " args_activation_size args_output_size \n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 8000000 400000 \n", + "4 NaN NaN \n", + "... ... ... \n", + "2550 NaN NaN \n", + "2551 3200000 800000 \n", + "2552 NaN NaN \n", + "2553 NaN NaN \n", + "2554 NaN NaN \n", + "\n", + "[2555 rows x 14 columns]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = oinf.get_profiling(as_df=True)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAEICAYAAAC3eUuqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtFElEQVR4nO3deZwU9Z3/8debwxkQAVFjUFwGE1Q0IHIYEVQiwWM1HjHeUfBcMSa6iZuYTX6uMSaruVZNXKPxXGMUBTUeiaIoq+LFIQ5egAeuIIkKiICAHJ/fH1VDGhxmGuiemp56Px+Pfkx1ddW3PkX3p/n0t75VpYjAzMzMLE9aZR2AmZmZWVNzAWRmZma54wLIzMzMcscFkJmZmeWOCyAzMzPLHRdAZmZmljsugMw2gqRbJF3WRNs6WdK4ptiWmVneuAAyawYk1UgKSW3q5kXE7RFxUJZxmZm1VC6AzJqApNZZx2BmTavwB401Py6AzBogaS9JUyUtljQaqE7nj5T09HrLhqQvptO3SLpW0l8kLQW+IukwSS9K+ljSu5IuKVj9yfTvR5KWSBq0/jYk7StpkqRF6d99C16bIOmnkiamsY6TtG2Z/lnMMiNptqQLJdWmuTBaUvUGlm0l6ceS3pH0vqT/kdSp4PUhkp6R9FGakyPT+e0k/Tpdb5Gkp9N5QyXNqSeer6bTl0gaI+mPkj4GRkrqJOlGSfMkzZV0Wd0Porocl/QrSQslvS3p0IK2u0i6WdJ76ev3Fbx2uKRpaezPSOpT8NoP0m0tljRD0rCS/OO3MC6AzDZA0hbAfcBtQBfgbuCYjWjiJOBnwFbA08BS4FSgM3AYMErSUemy+6d/O0dEh4h4dr1YugAPAVcD2wC/AR6StM162zsN+BywBXDhRsRqVkmOAw4BegB9gJEbWG5k+vgKsDPQAfgdgKTuwF+B3wLbAX2Bael6vwL6A/uS5P73gTVFxnYkMIYkz28HbgFWAV8E9gIOAs4sWP7LwAxgW+AXwI2SlL52G9Ae2IMkr/8rjX0v4CbgX0i+D64D7pdUJWlX4DxgYERsBRwMzC4y9lxxAWS2YfsAbYErI2JlRIwBJm3E+n+OiIkRsSYilkfEhIiYnj6vBe4ADiiyrcOAWRFxW0Ssiog7gNeBrxUsc3NEzIyIZcBdJF/oZi3R1RHxXkQsAB5gw5/1k4HfRMRbEbEE+CFwQnpo6iTgsYi4I83v+RExTVIr4HTg/IiYGxGrI+KZiFhRZGzPRsR9EbEG6Aj8M3BBRCyNiPdJipgTCpZ/JyL+EBGrgVuBrsD2kroChwLnRMTCNMb/Tdc5G7guIp5P47sVWEHynbUaqAJ2l9Q2ImZHxJtFxp4rLoDMNmwHYG6se8fgdzZi/XcLn0j6sqQnJH0gaRFwDsmvvmJjWX/b7wA7Fjz/W8H0JyS/ds1aomI/6+vnzTtAG2B7YCegvsJgW5JD3ZtaNBTmfXeSH1Hz0kNVH5H01nyuYJm1+xIRn6STHdL4FkTEwnq20R34Xl2babs7ATtExBvABcAlwPuS7pS0wybuS4vmAshsw+YBOxZ0RwP8U/p3KUnXNACSPl/P+rHe8z8B9wM7RUQn4PeANrDs+t4j+dIr9E/A3EbWM8uz9fPmn0gOR/2dpFD5Qj3rfAgs38Br6+d9a5LDZ4UKc/ldkp6ZbSOic/roGBF7FBH7u0AXSZ038NrPCtrsHBHt055hIuJPETGEZN8DuKKI7eWOCyCzDXuW5MvyO5LaSvo6sHf62kvAHpL6pgMwLymiva1IftEtl7Q3SRd8nQ9IxhjsvIF1/wLsIukkSW0kHQ/sDjy40Xtllh93AP8qqYekDsDPgdERsYpkfM5XJR2X5tQ2kvqmh65uAn4jaQdJrdOTEqqAmUC1khMa2gI/JjncVK+ImAeMA34tqWM6KPsLkho99J2u+1fgvyVtnX4H1Y0V/ANwTtqrLElbpjFtJWlXSQem8S4HllH8+KVccQFktgER8SnwdZJBlAuA44F70tdmApcCjwGzSAY5N+Zc4FJJi4GLScbp1G3rE5IB0xPTLu191otlPnA48D1gPsmgzMMj4sPN2EWzlu4mkoHETwJvkxQE3waIiP8jGZ/zPZL8ngbsma53ITCdZMzfApIelFYRsYgkj28g6X1dCqxzVlg9TiU5KeFVYCHJAOmuRcZ/CrCSZLzf+ySHtoiIycBZJAO6FwJv8I+B4FXA5SQ9WX8jOdz2wyK3lytad3iDmZmZWcvnHiAzMzPLHRdAZmZmljsugMzMzCx3XACZmZlZ7vhGbTmx7bbbRk1NTdZhmDV7U6ZM+TAi1r+2S7PifDYrTkP57AIoJ2pqapg8eXLWYZg1e5I25mrfmXA+mxWnoXz2ITAzMzPLHRdAZmZmljsugMzMzCx3XACZmZlZ7rgAMjMzs9xxAWRmZma54wLIzMzMcsfXAcqJ6XMXUXPRQ1mHYZap2ZcflnUIJeF8trwrRS67B8jMzMxyxwWQmZmZ5Y4LIDMzM8sdF0BmZmaWOy6AzMzMLHdcADUTko6SFJJ228DrEyQNqGf+SEm/K3+EZlYs57NZ8+cCqPk4EXg6/Wtmlc35bNbMuQBqBiR1AIYAZwAnpPPaSbpT0muS7gXaFSx/mqSZkl4ABmcStJnVy/lsVhl8IcTm4Ujg4YiYKWm+pP7AAcAnEdFLUh9gKoCkrsBPgP7AIuAJ4MX6GpV0NnA2QOuO25V/L8wMnM9mFcE9QM3DicCd6fSd6fP9gT8CREQtUJu+/mVgQkR8EBGfAqM31GhEXB8RAyJiQOv2ncoWvJmtw/lsVgHcA5QxSV2AA4HekgJoDQQb+BVoZs2X89mscrgHKHvfAG6LiO4RURMROwFvA1OAkwAkfQnoky7/PHCApG0ktQWOzSJoM6uX89msQrgHKHsnAlesN28ssBfQTtJrwGskX6BExDxJlwDPAh8B05oqUDNrlPPZrEK4AMpYRHylnnlXN7LOzcDNZQvKzDaJ89mscvgQmJmZmeWOCyAzMzPLHRdAZmZmljsugMzMzCx3PAg6J3rv2InJlx+WdRhmVgLOZ7PN5x4gMzMzyx0XQGZmZpY7LoDMzMwsd1wAmZmZWe64ADIzM7PccQFkZmZmueMCyMzMzHLHBZCZmZnljgsgMzMzyx0XQGZmZpY7LoDMzMwsd1wAmZmZWe64ADIzM7PccQFkZmZmueMCyMzMzHLHBZCZmZnljgsgMzMzy502WQdgTWP63EXUXPRQ1mGYbbbZlx+WdQiZcz5nz5/DyuceIDMzM8sdF0BmZmaWOy6AzMzMLHdcAJmZmVnuuAAyMzOz3HEBVCKSjpIUknbbhHUnSPo/SSqYd5+kJen0DpLGlDJeM9sw57NZy+cCqHROBJ5O/65DUjGXG/gIGJwu3xnoWvdCRLwXEd8oSZRmVgzns1kL5wKoBCR1AIYAZwAnpPOGSnpK0v3Aq5JaSfpvSa9LelTSXyQVfgneWbcu8HXgnoL2ayS9nE6PlHSPpIclzZL0i6bYR7McaYXz2azFcwFUGkcCD0fETGC+pP7p/H7A+RGxC8mXYA2wO3AKMGi9NsYD+0tqTfLFObqB7fUFjgd6A8dL2qm+hSSdLWmypMmrP1m0STtmlkOdcT6btXgugErjRJJffKR/67rNX4iIt9PpIcDdEbEmIv4GPLFeG6tJutxPANpFxOwGtjc+IhZFxHLgVaB7fQtFxPURMSAiBrRu32mjd8osp7rgfDZr8XwrjM0kqQtwINBbUgCtgQAeApZuZHN3AvcClzSy3IqC6dX4fTQriQULFgBsBdzgfDZr2dwDtPm+AdwWEd0joiYidgLeBvZbb7mJwDHp2IHtgaH1tPUU8J/AHeUM2MzqN2bMGIAFzmezls8F0OY7keRXXqGxfPbskbHAHJIu7j8CU4F1DuRH4lcR8WGZYjWzBtxxxx0AC9eb7Xw2a4EUEVnHkBuSOkTEEknbAC8Ag9PxA2VX1bVndB1xZVNsyqysyn0XbklTImJAEcs5n3PMd4OvDA3ls481N60H02uCbAH8tKm+LM2sLJzPZhXMBVATioihWcdgZqXhfDarbB4DZGZmZrnjHqCc6L1jJyb7mLVZi+B8Ntt87gEyMzOz3HEBZGZmZrnjAsjMzMxyxwWQmZmZ5Y4LIDMzM8sdF0BmZmaWOy6AzMzMLHdcAJmZmVnuuAAyMzOz3HEBZGZmZrnjAsjMzMxyp9ECSFJ7Sf9P0h/S5z0lHV7+0MyslD755BN++tOfctZZZwEwa9YsHnzwwYyjMjPLRjE9QDcDK4BB6fO5wGVli8jMyuK0006jqqqKZ599FoAdd9yRH//4xxlHZWaWjWIKoC9ExC+AlQAR8QmgskZlZiX35ptv8v3vf5+2bdsC0L59eyIi46jMzLJRTAH0qaR2QABI+gJJj5CZVZAtttiCZcuWISW/X958802qqqoyjsrMLBttiljmP4CHgZ0k3Q4MBkaWMygzK72f/OQnHHLIIbz77rucfPLJTJw4kVtuuSXrsMzMMtFoARQRj0qaCuxDcujr/Ij4sOyRmVlJDR8+nH79+vHcc88REVx11VVsu+22WYdlZpaJYnqAAHYEWqfL7y+JiLinfGFZqU2fu4iaix7KOgxbz+zLD2vS7c2dO5fVq1ezatUqnnzySQC+/vWvN2kMtvmcz9lr6ty10mu0AJJ0E9AHeAVYk84OwAWQWQU5/fTTqa2tZY899qBVq2T4nyQXQGaWS8X0AO0TEbuXPRIzK6vnnnuOV199NeswzMyahWLOAntWkgsgswo3aNAgF0BmZqlieoD+h6QI+hvJ6e8CIiL6lDUyMyupU089lUGDBvH5z3+eqqoqIgJJ1NbWZh2amVmTK6YAuhE4BZjOP8YAmVmFOeOMM7jtttvo3bv32jFAZmZ5VUwB9EFE3L+xDUtaTVI0tQHeBk6JiI82Yv0lEdFhY7fbQHutgCuBA0kGcS8HjouIt0vQ9unAv6bttgJ+FBF/3tx2zUppu+2244gjjtikdVu3bk3v3r1ZtWoVPXr04LbbbqNz585Fr9+hQweWLFmySduuz5o1a7jgggt4/PHHkUR1dTV33XUXPXr0KEXz20iajvPZrEUrpgB6UdKfgAcouAJ0EafBL4uIvgCSbgW+BfxsE+MsheOBHYA+EbFGUjdg6eY2mrbzI6BfRCyS1AHYbnPbbWSbrSNidTm3YS3PXnvtxUknncTXvva1da4AXcxZYO3atWPatGkAjBgxgmuuuYYf/ehH5Qq1UaNHj+a9996jtraWVq1aMWfOHLbccsvNbnfOnDkAXYHtnc9mLVsx/eDtSAqfg4CvpY+NvRv8syTXEkLSFyQ9LGmKpKck7ZbO7yHpWUnTJa292aqkoZIeLHj+O0kj0+mBkp6R9JKkFyRtJam1pF9KmiSpVtK/pKt2BeZFxBqAiJgTEQvTdpYUtP8NSbek07dIulbSc5LeSmO5SdJrdcsAnwMWA0vSdpfU9SpJ6p/G9lIa08vp/JGSflewzQclDU2nr5U0WdIrkn5SsMxsSVekF6U8VtJB6b/XVEl3p1/UZhu0bNkyqqqqGDduHA888AAPPPDAJt0NftCgQcydOxdIbqdxyCGH0L9/f/bbbz9ef/11AN5++20GDRpE796917nh6oQJEzj88H98fZx33nlrr0Y9adIk9t13X/bcc0/23ntvFi9ezOrVq/m3f/s3Bg4cSJ8+fbjuuusAmDdvHl27dl17KK9bt25svfXWQNLbVGfMmDGMHDkSgJEjRzJq1Cj22Wcfdt55ZyZMmMDpp59Or1691i7z/vvvQ3Ko3/ls1sIVcyXo0zZnA5JaA8NIxhIBXA+cExGzJH0Z+G+Sw1JXAddGxP9I+lYR7W4BjAaOj4hJkjoCy4AzgEURMVBSFTBR0jjgLuBpSfsB44E/RsSLRezC1sAg4AjgfpJbgZwJTJLUF3gJ+DvwtqTxwD0R8UC67s3AeRHxpKRfFrEtSLrbF6T/buMl9YmIulGq8yOin6RtSa7D9NWIWCrpB8B3gUvX+zc6GzgboHXHsv6ItQpw8803b3Ybq1evZvz48ZxxxhkAnH322fz+97+nZ8+ePP/885x77rk8/vjjnH/++YwaNYpTTz2Va665ptF2P/30U44//nhGjx7NwIED+fjjj2nXrh033ngjnTp1YtKkSaxYsYLBgwdz0EEHcdxxxzFkyBCeeuophg0bxje/+U322muvRrezcOFCnn32We6//36OOOIIJk6cyA033MDAgQOZNm0ae+65JyQ3fnY+m7VwxVwIsZqkqNgDqK6bHxGnN7JqO0nTSHp+XgMeTX/V7AvcLa29oXxdX/xg4Jh0+jbgikba35WkR2dSGs/HabwHAX0kfSNdrhPQMyLGSdqVpNg6kOTL6NiIGN/Idh6IiEjHBPw9Iqan23kFqImIaZIOAQaSFHr/Jak/yXijzhHxZME+HdrItgCOS7/o2pD0Wu0O1H1hjk7/7pPOn5j+O25B0su2joi4nqTgpKprT9/2O+eWL1/OjTfeyCuvvMLy5cvXzr/pppsaXXfZsmX07duXuXPn0qtXL4YPH86SJUt45plnOPbYY9cut2JFcpR84sSJjB07FoBTTjmFH/zgBw22P2PGDLp27crAgQMB6NixIwDjxo2jtraWMWPGALBo0SJmzZrFQQcdxIwZM3j88cd5/PHHGTZsGHfffTfDhg1rcDtf+9rXkETv3r3Zfvvt6d27NwB77LEHs2fPpm/fvgCzgHNxPpu1aMWMAboNeB04mOQXyckkBU1jlkVEX0ntgUdIxgDdAnxUNzaoHvUl9SrWPVRXXc8yhQR8OyIe+UzjESuAvwJ/lfR34CiS3qDC7a7fft24pzUF03XP26TtBvAC8IKkR0l+KV7ZQIz17pOkHsCFwMCIWJgeZiuMp27MkoBHI+LEBrZhto5TTjmF3XbbjUceeYSLL76Y22+/nV69ehW1bt0YoE8++YSDDz6Ya665hpEjR9K5c+e1Y4PWV/AjZ602bdqwZs0/TiYtLMTqExH89re/5eCDD/7Ma1VVVRx66KEceuihbL/99tx3330MGzZsne2u337d2KdWrVqtMw6qVatWrFq1qnC7zmezFq6YMUBfjIj/ByyNiFuBw4AvF7uBiPgE+A7wPeATkq7lYwGU2DNddCJwQjp9ckET7wC7S6qS1JnkVxnADKCrpIFpW1tJakNSbI2S1Dadv4ukLSX1k7RDOq8Vye093knb+rukXun8o4vdt7StHST1K5jVF3gnPePtI0lD6tmn2UBfSa0k7QTsnc7vSPKluEjS9mz4F+ZzwGBJX0xj2FLSLhsTt+XPG2+8wU9/+lO23HJLRowYwUMPPcTzzz+/UW20b9+eq6++ml//+te0b9+eHj16cPfddwNJsfLSSy8BMHjwYO68804Abr/99rXrd+/enVdffZUVK1bw0UcfMX580gG76667Mm/ePCZNmgTA4sWLWbVqFQcffDDXXnstK1euBGDmzJksXbqUqVOn8t577wHJGWG1tbV0794dgO23357XXnuNNWvWcO+9927U/qVtti+Y1Rfns1mLVEwP0Mr070eSvgT8jWTgb9Ei4kVJtcCJJF8c10r6MdAWuJNkHM35wJ/S499/Llj3XUl3AS+TnE7/Yjr/U0nHA7+V1I5k/M9XgRuAGmCqkp+CH5D09HwO+EM6LgiSHpu6gYsXAQ+my04GNmYAYlvgV2lxtTxt45z0tdOAmyQFMK5gnYnpvrxK0ps2Nd2nlyS9SNLj9m663GdExAdKBoLfUbA/PwZmbkTcljNt27YFoHPnzrz88st8/vOfrxv0u1H22msv+vTpwx133MHtt9/OqFGjuOyyy1i5ciUnnHACe+65J1dddRUnnXQSV1xxBUceeeTadXfaaSeOO+44vvSlL9GjR4+143a22GILRo8ezbe//W2WLVtGu3bteOyxxzjzzDOZPXs2/fr1IyLYbrvtuO+++3j//fc566yz1h5y23vvvTnvvPMAuPzyyzn88MPZbrvtGDBgwEadfp8WWt0kvY7z2axFU3L0poEFpDOBsSQ9JjeTFAcXR8Tvyx9eyyGpBngwIr6UxfaruvaMriOuzGLT1oCmvKP0DTfcwDHHHENtbS2nnXYaS5Ys4dJLL+Wcc85pfOUckTQlIgY0skwNzudc893gK0ND+VzMWWA3pJP/C+xcysDMrOmceeaZABxwwAG89dZbGUdjZpatYs4CqyI5O6umcPmIuHRD69hnRcRsIJNfi2aQnKE1duxYZs+evc6A34svvjjDqCqT89ms8hUzBujPwCJgCuueBWVmFeTII4+kU6dO9O/ff50zoMzM8qiYAqhbRBxS9kisrHrv2InJPmada3PmzOHhhx/OOgwrAeez2eYr5jT4ZyT1LnskZlZW++67L9OnT886DDOzZqGYHqAhwEhJb5McAhPJtf/6lDUyMyupp59+mltuuYUePXpQVVVFRCCJ2traxlc2M2thiimAGrzcu6St624qambN11//+tcGX1+4cOHaG4qambV0xZwG/04ji4wH+jWyjJllrO5KyRsybNgwpk6d2kTRmJllq5gxQI357A1/zKziNHZRVDOzlqQUBZC/Nc1agPpuXmpm1lKVogAyMzMzqyg+BGZmgA+BmVm+FHMWGJL6kZwOH8DEiCgcKTmsHIGZWelNnTqVp59+GkkMHjyYfv3+cf7C+PHjM4zMzKxpNdoDJOli4FZgG2Bb4GZJP657PSIWlC88MyuVSy+9lBEjRjB//nw+/PBDTjvtNC677LK1r3fp0iXD6MzMmpYa6/aWNAPYMyKWp8/bAdMiYtcmiM9KZMCAATF58uSsw7AM7brrrrz00ktUV1cDsGzZMvr27cuMGTMyjqx5kTQlIgZkHUdDnM9mxWkon4sZA/QeUF3wvAqYW4rAzKzp7LDDDixfvnzt8xUrVrDjjjtmGJGZWXaKGQO0CHhF0qMkY4CGAy9IuhogIr5TxvjMrEQ6derEHnvswfDhw5HEo48+yt577813vpOk8NVXX51xhGZmTaeYAuje9FFnQnlCMbNyOvroozn66KPXPh86dGh2wZiZZayYW2HcKmkLYJd01oyIWFnesMys1EaMGMGnn37KzJkzgWRMUNu2bTOOyswsG40WQJKGkpwFNpvkmj87SRoREU+WNTIzK6kJEyYwYsQIampqiAjeffddbr31Vvbff/+sQzMza3LFHAL7NXBQRMwAkLQLcAfQv5yBmVlpfe9732PcuHHsumtyAufMmTM58cQTmTJlSsaRmZk1vWIKoLZ1xQ9ARMyU5H7zCjN97iJqLnoo6zByZ/blh2UdwlorV65cW/wA7LLLLqxc6aPZlcj5nL3mlNu2aYopgCZLugH4Y/r8ZMAXoDCrMAMGDODMM8/km9/8JgC33347AwY068vdmJmVTTEF0CjgW0Dd6e5PAf9dtojMrCyuvfZarrnmmrWnu++3336ce+65GUdlZpaNYs4CWwH8Jn18hqSxEXFMqQMzs9Kqqqriu9/9Lt/97nfrff2YY45h7NixTRyVmVk2SnE3+J1L0IaZZeytt97KOgQzsyZTigKo4ZuJmVlFkJR1CGZmTaYUBZCViKSQ9MeC520kfSDpwSLWXVLe6MysWM5ls+avFAWQfzaWzlLgS5Lapc+H4xvPWhOJcGduCTmXzZq5jSqAJG0tqc96s39QwngM/gLUXWDiRJKLTgIg6RJJFxY8f1lSTdOGZy3BwoULqa2tXWfeFVdckVE0LZZz2awZa7QAkjRBUkdJXYCpwB8krT0jLCLGlTPAHLoTOEFSNdAHeH5TG5J0tqTJkiav/mRRyQK0yjR06FA+/vhjFixYQL9+/TjrrLPWOSPsoIMOyjC6FqlkuQzOZ7NSK6YHqFNEfAx8HfifiPgy8NXyhpVfEVEL1JD8YvzLZrZ1fUQMiIgBrdt3KkV4VsEWLVpEx44dueeeezj11FN5/vnneeyxx7IOq8UqZS6n7TmfzUqomAKojaSuwHFAowP4rCTuB35FQZd5ahXrvmfVTRaRVbxVq1Yxb9487rrrLg4//PCsw8kL57JZM1VMAXQp8AjwRkRMkrQzMKu8YeXeTcBPImL6evNnA/0AJPUDejRxXFbBLr74Yg4++GC++MUvMnDgQN566y169uyZdVgtnXPZrJkq5krQdwN3Fzx/C/CVn8soIuYAV9fz0ljgVEmvkIwnmNmkgVlFO/bYYzn22GPXPt9555195ecycy6bNV+NFkCS6kveRcDkiPhz6UPKr4joUM+8CcCEdHoZUO9I1frWNSv0ne985zPzOnXqxIABAzjyyCMziKjlci6bNX/FHAKrBvqSHPaaRXI2QzfgDElXli0yMyup5cuXM23aNHr27EnPnj2pra1lzpw53HjjjVxwwQVZh2dm1qSKuRt8H2BwRKwGkHQtyR3hhwDrH9c2s2aqtraWiRMn0rp1awBGjRrFfvvtx9NPP03v3r0zjs7MrGkV0wO0NVDYJbsl0CUtiFaUJSozK7mFCxeyZMk/7rKwdOlSFixYQOvWramqqsowMjOzpldMD9AvgGmSJpDc9mJ/4OeStgR8EZEK0XvHTky+/LDGF7QW6/vf/z59+/Zl6NChRARPPvkk//7v/87SpUv56ld9aa9K4nw223wNFkCSWgGvAfsCe6ez/z0i3kun/62MsZlZiaxZs4ZevXrxzDPP8MILLwDw85//nB122AGAX/7yl1mGZ2bW5BosgCJijaRrImIvwGd8mVWoVq1a8a1vfYsXX3zRZ3yZmVHcGKDxko6R5Lu+m1WwYcOGMXbsWN/13cyM4gqgfyG5EOIKSR9LWizp4zLHZWYldt1113HsscdSVVVFx44d2WqrrejYsWPWYZmZZaKYK0Fvld4Jvie+X41ZxVq8eDELFixg1qxZLF++POtwzMwyVcyVoM8Ezie5+OE0YB/gGWBYWSMzs5K64YYbuOqqq5gzZw59+/blueeeY99992X8+PFZh2Zm1uSKOQR2PjAQeCcivgLsRXIrDDOrIFdddRWTJk2ie/fuPPHEE7z44ot06tQp67DMzDJRTAG0PCKWA0iqiojXgV3LG5aZlVp1dTXV1clR7BUrVrDbbrsxY8aMjKMyM8tGMRdCnCOpM3Af8KikhcA75QzKzEqvW7dufPTRRxx11FEMHz6crbfemu7du2cdlplZJooZBH10OnmJpCeATsDDZY3KzEru3nvvBeCSSy7hK1/5CosWLeKQQw7JOCozs2wU0wO0VkT8b7kCMbOmc8ABB2QdgplZpooZA2RmZmbWorgAMjMzs9xxAWRmZma54wLIzMzMcscFkJmZmeWOCyAzMzPLnY06Dd4q1/S5i6i56KGsw2gysy8/LOsQzMomb/ncHPk7pvK5B8jMzMxyxwWQmZmZ5Y4LIDMzM8sdF0BmZmaWOy6AzMzMLHdcAJWYpO0l/UnSW5KmSHpW0tFZx2VmG8e5bNayuQAqIUkC7gOejIidI6I/cALQLdPAzGyjOJfNWj4XQKV1IPBpRPy+bkZEvBMRv5XUWtIvJU2SVCvpXwAkDZX0v5L+nP7SvFzSyZJekDRd0hfS5W6RdK2k59Llhkq6SdJrkm7JZnfNWiznslkL5wKotPYApm7gtTOARRExEBgInCWpR/ransA5QC/gFGCXiNgbuAH4dkEbWwODgH8F7gf+K91mb0l919+gpLMlTZY0efUnizZ338zypFnlMjifzUrNV4IuI0nXAEOAT4F3gD6SvpG+3Anomb42KSLmpeu8CYxLl5kOfKWgyQciIiRNB/4eEdPTdV4BaoBphduPiOuB6wGquvaMUu+fWV5kncvgfDYrNRdApfUKcEzdk4j4lqRtgcnA/wHfjohHCleQNBRYUTBrTcHzNaz7Hq2oZ5n6ljOzzeNcNmvhfAistB4HqiWNKpjXPv37CDBKUlsASbtI2rKpAzSzojiXzVo4/9IoobRL+yjgvyR9H/gAWAr8ALibpGt7anqGyQfAUdlEamYNcS6btXyK8KHkPKjq2jO6jrgy6zCajO/UbJtK0pSIGJB1HA3JWz43R/6OqQwN5bMPgZmZmVnuuAAyMzOz3HEBZGZmZrnjAsjMzMxyx2eB5UTvHTsx2YP2zFoE57PZ5nMPkJmZmeWOCyAzMzPLHRdAZmZmljsugMzMzCx3XACZmZlZ7rgAMjMzs9xxAWRmZma54wLIzMzMcscFkJmZmeWOCyAzMzPLHRdAZmZmljsugMzMzCx3XACZmZlZ7rgAMjMzs9xxAWRmZma54wLIzMzMcscFkJmZmeVOm6wDsKYxfe4iai56qOTtzr78sJK3aWYNK1c+W/H83Vf53ANkZmZmueMCyMzMzHLHBZCZmZnljscAmVlRVq5cyZw5c1i+fHnWoZREdXU13bp1o23btlmHYtbknM8ugMysSHPmzGGrrbaipqYGSVmHs1kigvnz5zNnzhx69OiRdThmTc757ENgmZP0I0mvSKqVNE3Slzdi3b6S/rmc8ZnVWb58Odtss03Ff1kCSGKbbbYp+a9f57NVCueze4AyJWkQcDjQLyJWSNoW2KLIddsAfYEBwF/KFqRZgZbwZVmn1PvifLZKk/d8dgGUra7AhxGxAiAiPgSQdAhwJfAJ8DSwc0QcLukS4AvAzsD/AYOBdpKGAP8ZEaObfA/MrI7z2ayCuADK1jjgYkkzgceA0cDzwB+AA4E30nmFdgeGRMQySSOBARFxXn2NSzobOBugdcftyrIDll+lvhDfplxY7pJLLqFDhw5ceOGFJY1lEzmfrWJlnc9Z5LLHAGUoIpYA/Um+1D4g+XI8B3g7ImZFRAB/XG+1+yNiWZHtXx8RAyJiQOv2nUoZulnFWbVqVVnbdz6bNY1S5bILoIxFxOqImBAR/wGcBwxrZJWlTRCWWbP1s5/9jF122YUhQ4YwY8YMAIYOHcrkyZMB+PDDD6mpqQHglltu4YgjjuDAAw9k2LDGUmvzOZ/Nipd1LvsQWIYk7QqsiYhZ6ay+wN+BPSV9ISLeBE5soInFwFbljdKs+ZgyZQp33nkn06ZNY9WqVfTr14/+/fs3uM7UqVOpra2lS5cuZY3N+WxWvOaQy+4BylYH4FZJr0qqJRkPcBFJF/pDkqYC7zew/hPA7unptseXP1yzbD311FMcffTRtG/fno4dO3LEEUc0us7w4cPLXvyknM9mRWoOueweoAxFxBRg33peehjYDUDSUODCdPlL1lt/ATCwnDGaVYI2bdqwZs0agM9cC2TLLbdskhicz2abrylz2T1AZlYx9t9/f+677z6WLVvG4sWLeeCBBwCoqalhypQpAIwZMybLEM2sCM0hl90D1MxFxARgQsZhmH3Gppy2vrn69evH8ccfz5577snnPvc5Bg5MOkwuvPBCjjvuOK6//noOO6zp4yqW89maq6bO5+aQy0rOzLSWrqprz+g64sqSt5vFf4KWjddee41evXplHUZJ1bdPkqZExICMQipKufLZilfp333OZ/cA5UbvHTsxucIT1swSzmezzecxQGZmZpY7LoDMrGgt6ZB5S9oXs03RknJgU/bFBZCZFaW6upr58+e3iC/NiGD+/PlUV1dnHYpZJpzPHgNkZkXq1q0bc+bM4YMPPsg6lJKorq6mW7duWYdhlgnnswsgMytS27Zt6dGjR9ZhmFkJOJ99CMzMzMxyyAWQmZmZ5Y4LIDMzM8sdXwk6JyQtBmZkHccm2Bb4MOsgNkElxl2JMUPp4+4eEduVsL2Sq+B83liV+pncWN7P8tlgPnsQdH7MaO6X96+PpMmOu2lUYsxQuXFvporM542Vl/fW+5kNHwIzMzOz3HEBZGZmZrnjAig/rs86gE3kuJtOJcYMlRv35sjLPns/W5ZmtZ8eBG1mZma54x4gMzMzyx0XQGZmZpY7LoByQNIhkmZIekPSRU20zZskvS/p5YJ5XSQ9KmlW+nfrdL4kXZ3GVyupX8E6I9LlZ0kaUTC/v6Tp6TpXS1JD29iIuHeS9ISkVyW9Iun85h67pGpJL0h6KY35J+n8HpKeT7czWtIW6fyq9Pkb6es1BW39MJ0/Q9LBBfPr/QxtaBsb+W/eWtKLkh6spLizkEUul0spc60SlOJz3txJ6ixpjKTXJb0maVCzfj8jwo8W/ABaA28COwNbAC8BuzfBdvcH+gEvF8z7BXBROn0RcEU6/c/AXwEB+wDPp/O7AG+lf7dOp7dOX3shXVbpuoc2tI2NiLsr0C+d3gqYCezenGNP2+mQTrcFnk/bvws4IZ3/e2BUOn0u8Pt0+gRgdDq9e/r5qAJ6pJ+b1g19hja0jY38N/8u8CfgwYbabG5x5yWXy7g/Jcm1Snls7ue8Eh7ArcCZ6fQWQOfm/H5m/g/mR5nfYBgEPFLw/IfAD5to2zWsWwDNALqm011JLuYGcB1w4vrLAScC1xXMvy6d1xV4vWD+2uU2tI3N2Ic/A8MrJXagPTAV+DLJFVfbrP85AB4BBqXTbdLltP5no265DX2G0nXq3cZGxNsNGA8cCDzYUJvNKe4sHhvan6zjKuH+bVKuZR13kfu22Z/zrPehiH3sBLy9fqzN+f30IbCWb0fg3YLnc9J5Wdg+Iual038Dtk+nNxRjQ/Pn1DO/oW1stLTreS+SHpVmHXvavT4NeB94lKSn4KOIWFXPdtbGlr6+CNhmE/Zlmwa2Uawrge8Da9LnDbXZnOLOQnPK5ZLazFyrBFey+Z/z5q4H8AFwc3qo7wZJW9KM308XQJaJSEr+sl6DYXO2IakDMBa4ICI+LlW7xdrYbUTE6ojoS/JLc29gtzKFVjKSDgfej4gpWcdi2ck618otR5/zNiTDHq6NiL2ApSSHvNZqbu+nC6CWby6wU8Hzbum8LPxdUleA9O/76fwNxdjQ/G71zG9oG0WT1JbkC/n2iLinkmKPiI+AJ0i61DtLqrvfX+F21saWvt4JmL8J+zK/gW0UYzBwhKTZwJ0khweuqoC4s9KccrkkSpRrzV2pPufN3RxgTkQ8nz4fQ1IQNdv30wVQyzcJ6JmecbAFyaC6+zOK5X6g7myoESTH/Ovmn5qeFbAPsCjtMn0EOEjS1umZAweRHCefB3wsaR9JAk5dr636tlGUtL0bgdci4jeVELuk7SR1TqfbkYyjeI2kEPrGBmKu2843gMfTX2b3AyekZ6H0AHqSDNiu9zOUrrOhbTQqIn4YEd0ioiZt8/GIOLm5x52h5pTLm62EudaslfBz3qxFxN+AdyXtms4aBrxKc34/sxow5UfTPUhG288kGRfyoyba5h3APGAlyS+DM0iOY48HZgGPAV3SZQVck8Y3HRhQ0M7pwBvp47SC+QOAl9N1fsc/rmpe7zY2Iu4hJF20tcC09PHPzTl2oA/wYhrzy8DF6fydSQqBN4C7gap0fnX6/I309Z0L2vpRGtcM0rPTGvoMbWgbm/B5Gco/zo6pmLjzkMtl3JeS5VqlPDb3c97cH0BfYHL6nt5HcgZss30/fSsMMzMzyx0fAjMzM7PccQFkZmZmueMCyMzMzHLHBZCZmZnljgsgMzMzyx0XQGZmZpY7LoDMzMwsd/4/IY+YzwsleUsAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "gr_dur = df[['dur', \"args_op_name\"]].groupby(\"args_op_name\").sum().sort_values('dur')\n", + "gr_n = df[['dur', \"args_op_name\"]].groupby(\"args_op_name\").count().sort_values('dur')\n", + "gr_n = gr_n.loc[gr_dur.index, :]\n", + "\n", + "fig, ax = plt.subplots(1, 2, figsize=(8, 4))\n", + "gr_dur.plot.barh(ax=ax[0])\n", + "gr_n.plot.barh(ax=ax[1])\n", + "ax[0].set_title(\"duration\")\n", + "ax[1].set_title(\"n occurences\");" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArwAAAD4CAYAAAAO7DUQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAYa0lEQVR4nO3df5ScZX338fe3SSRAAEn4lYdQNiDyowIxiYpIcygRBIMBNPJDKyAeqT5t1fbEHnrQQnsejtg+bS2PbTW2GsQfsUDRUI4C/qrYAppASIQQAcEaiiKxQhQCgXyfP+5rYbLsbmaTnZ3da9+vc+bszDX3fd3fa+7dzCfXXDMTmYkkSZJUq9/odgGSJElSJxl4JUmSVDUDryRJkqpm4JUkSVLVDLySJEmq2sRuF6DRba+99sqenp5ulyFJkrRNK1eufCwz9+7bbuDVoHp6elixYkW3y5AkSdqmiPhxf+0uaZAkSVLVDLySJEmqmoFXkiRJVXMNryRJUuU2b97M+vXr2bRpU7dLGRaTJ09mxowZTJo0qa3tDbySJEmVW79+Pbvtths9PT1ERLfL2SGZyYYNG1i/fj0zZ85sax+XNEiSJFVu06ZNTJs2bcyHXYCIYNq0aUOarTbwSpIkjQM1hN1eQx2LgVeSJElVcw2vJEnSONNz0Q3D2t9Dly8Y0vaXXnopU6ZMYfHixcNax0Cc4ZUkSdKo9eyzz+5wHwZeSZIkddxll13Gy1/+co477jjWrVsHwPHHH8+KFSsAeOyxx+jp6QFg6dKlLFy4kBNOOIH58+fv8LFd0iBJkqSOWrlyJcuWLWPVqlU8++yzzJ49mzlz5gy6zx133MHq1auZOnXqDh/fwCtJkqSOuuWWWzjjjDPYZZddAFi4cOE29znxxBOHJeyCSxokSZLUJRMnTmTLli0AL/pc3V133XXYjmPglSRJUkfNmzePL3/5yzz11FNs3LiR66+/HoCenh5WrlwJwDXXXNOx47ukQZIkaZwZ6seI7ajZs2dz1llncfTRR7PPPvvwqle9CoDFixdz5plnsmTJEhYs6FxNkZkd61xj39y5c7P33ZOSJGlsWrt2LYcffni3yxhW/Y0pIlZm5ty+27qkQZIkSVUz8EqSJKlqBl5JkqRxoKZlrEMdi4FXkiSpcpMnT2bDhg1VhN7MZMOGDUyePLntffyUBkmSpMrNmDGD9evX8/Of/7zbpQyLyZMnM2PGjLa3N/BKkiRVbtKkScycObPbZXSNSxokSZJUNQOvJEmSqmbglSRJUtVcw6tBrXn4cXouuqHbZUiSpDFqpL/GuD/O8EqSJKlqBl5JkiRVzcArSZKkqhl4JUmSVDUDryRJkqpm4JUkSVLVDLySJEmqmoFXkiRJVTPwSpIkqWoGXkmSJFXNwCtJkqSqGXglSZJUNQOvJEmSqmbglSRJUtUMvJIkSaqagVeSJElVM/BKkiSpagZeSZIkVc3AK0mSpKoZeCVJklQ1A68kSZKqZuCVJElS1Qy8kiRJqlrXAm9EnB4RGRGHDXD/tyNi7jb6mFX6OHmQbZZGxKJ+2o+PiH8bZL9LI2LxYMcfThFxfkR8fJD7T4+II1pu/0VEvH5kqpMkSRq7ujnDew7w3fKzm30Mu4iY0IFuTweeD7yZ+WeZ+fUOHEeSJKkqXQm8ETEFOA54F3B2ads5IpZFxNqIuA7YeRt9BPBW4HzgxIiY3NseER+PiHUR8XVgn5Z9To6IeyPiDuDNQ6j33RHx1VLj70bE9yJiVUR8sjfcRsSvIuKvI+Iu4LXl9mURcVdE3BYR+5bt9o6IayPi++XyujaOfyywEPirctyDW2euI+KhiPhIuW9FRMyOiBsj4oGIeE9LPx8sx1wdEX/e7vglSZLGsm7N8J4GfC0zfwhsiIg5wHuBJzPzcOASYM42+jgWeDAzHwC+DSwo7WcAh9LMhp5btqME4k8Bbyp979dOoRHxB8CpNDOsPcBZwOsycxbwHPD2sumuwO2ZeXRmfrfcvi0zjwa+A7y7bPd3wN9m5quAtwD/tK0aMvM/geXABzNzVhlzX/9VaroFWAosAo4B/ryM4yTgEODVwCxgTkTMG2DMF5bgvOK5Jx/fVnmSJEmj2sQuHfccmuAHsKzcfhlwBUBmro6I1W30saylj3OBa4F5wBcz8zngvyPim2Wbw2gC8n0AEfE54MJtHONc4CfA6Zm5OSLm04Tl7zcTzOwMPFq2fa4cv9czQO8a4ZXAieX664Ejyv4Au5cZ7x21vPxcA0zJzI3Axoh4OiJeCpxULneW7abQBODv9O0oM5cASwB2mn5IDkNtkiRJXTPigTcipgInAEdGRAITgOSFINZOHxNoZkdPi4iLgQCmRcRuw1zuGprZ0BnAg+U4V2bmn/az7aYSsnttzszesPgcLzzWvwEck5mbWnduCcDb6+nyc0vL9d7bE0vtH8nMT+7ogSRJksaSbixpWARclZkHZmZPZh5AEyZXAm8DiIhXAEcN0sd8YHVmHlD6OJBmdvUMmhnLsyJiQkRMB36n7HMv0BMRB5fb7bzR7U7g94DlEfG/gG8AiyJin1Ln1Ig4sP2hA3AT8Ie9NyJiVpv7bQR2JNDfCFzQO5scEfv3jkOSJKlm3Qi85wDX9Wm7FpgJTImItcBf0ATgofbR234fcA/wWeBWgDKjeiFwQ3nT2qO0oazHXQzcUPb5EHBTWXJxMzC9nX5avA+YW944dg/wnm3tUCwDPhgRd7aE9rZl5k3AF4BbI2INcA07FqAlSZLGhHjhVXfpxXaafkhOP+9j3S5DkiSNUQ9dvmDbGw2TiFiZmS/6Hge/aU2SJElV69anNLQtIm4HdurT/I7MXDNM/b8TeH+f5v/IzN8fjv63o56LaT5fuNXVmXlZN+qRJEka60Z94M3M13S4/88An+nkMYaiBFvDrSRJ0jBxSYMkSZKqZuCVJElS1Qy8kiRJqpqBV5IkSVUz8EqSJKlqBl5JkiRVzcArSZKkqhl4JUmSVDUDryRJkqpm4JUkSVLVDLySJEmqmoFXkiRJVTPwSpIkqWoGXkmSJFXNwCtJkqSqGXglSZJUNQOvJEmSqmbglSRJUtUmdrsAjW5H7r8HKy5f0O0yJEmStpszvJIkSaqagVeSJElVM/BKkiSpagZeSZIkVc3AK0mSpKoZeCVJklQ1A68kSZKqZuCVJElS1Qy8kiRJqpqBV5IkSVUz8EqSJKlqBl5JkiRVzcArSZKkqhl4JUmSVDUDryRJkqrWVuCNiF0i4sMR8aly+5CIOLWzpUmSJEk7rt0Z3s8ATwOvLbcfBv5PRyqSJEmShlG7gffgzPxLYDNAZj4JRMeqkiRJkoZJu4H3mYjYGUiAiDiYZsZXkiRJGtUmtrndJcDXgAMi4vPA64DzO1WUJEmSNFzaCryZeXNE3AEcQ7OU4f2Z+VhHK5MkSZKGwVA+lmx/YALwEmBeRLy5MyVJkiRJw6etGd6I+DRwFHA3sKU0J/CvHapLkiRJGhbtruE9JjOP6GglkiRJUge0u6Th1ogw8EqSJGnMaXeG97M0ofenNB9HFkBm5lEdq0ySJEkaBu0G3n8G3gGs4YU1vJIkSdKo127g/XlmLu9oJZIkSVIHtBt474yILwDX0/INa5nppzRIkiRpVGs38O5ME3RPamnzY8kkSZI06rX7TWvv7HQhkiRJUie0+8UTk4F3Ab8FTO5tz8wLOlSXJEmSNCza/Rzeq4D9gDcA/w7MADZ2qihJkiRpuLQbeF+WmR8Gfp2ZVwILgNd0rixJkiRpeLQbeDeXn7+MiFcAewD7dKYkSZIkafi0+ykNSyJiT+BDwHJgCvDhjlUlSZIkDZN2A+9VwFuAHuDK0rZvJwqSJEmShlO7gfcrwOPASlq+eEKSJEka7doNvDMy8+SOViJJkiR1QLtvWvvPiDiyo5VIkiRJHdDuDO9xwPkR8SDNkoYAMjOP6lhlkiRJ0jBoN/Ce0tEqJEmSpA5pK/Bm5o87XYgkSZLUCe2u4ZUkSZLGJAOvJEmSqmbglSRJUtUMvJIkSaqagVeSJElVM/BKkiSpagZeSZIkVc3AK0mSpKoZeCVJklQ1A68kSZKq1tZXC2v8WvPw4/RcdEO3y5AkSaPQQ5cv6HYJbXGGV5IkSVUz8EqSJKlqBl5JkiRVzcArSZKkqhl4JUmSVDUDryRJkqpm4JUkSVLVDLySJEmqmoFXkiRJVTPwSpIkqWoGXkmSJFXNwCtJkqSqGXglSZJUNQOvJEmSqmbglSRJUtUMvJIkSaqagVeSJElVM/BKkiSpagZeSZIkVc3AK0mSpKoZeCVJklQ1A68kSZKqZuCVJElS1Qy8kiRJqlpHA29EnB4RGRGHDXD/tyNi7jb6mFX6OHmQbZZGxKJ+2o+PiH8r1w+LiFsj4umIWLyNY/ZExA8G22a4RcSvtlHP21puz42IK0amMkmSpLGt0zO85wDfLT+72QfAL4D3Af93B/sZVERM7EC3PcDzgTczV2Tm+zpwHEmSpOp0IpwBEBFTgOOA3wGuBy6JiJ2BzwBHA/cCO2+jjwDeCpwI3BIRkzNzU2n/f6X9J8AzLfucDHwMeJImKAOQmY8Cj0bEgiGO4yDgWuBCmtD898Depf93Z+a9EbEU2AS8EviPiJgKPAHMBfYD/iQzryn9fRA4E9gJuC4zL2mjjMuBwyNiFXAlcCewODNPjYhLgZnAQcBvAn8EHAOcAjwMvCkzN0fEHOBvgCnAY8D5mfnIAGO+sIyXCbvv3d4DJUmSNEp1cob3NOBrmflDYEMJXO8FnszMw4FLgDnb6ONY4MHMfAD4NtAbVs8ADgWOAM4t2xERk4FPAW8qfe+3IwOIiENpwu75mfl9YAnwh5k5B1gM/EPL5jOAYzPzj8vt6TSB/1SawEpEnAQcArwamAXMiYh5bZRyEXBLZs7KzL/t5/6DgROAhcDngG9l5pHAU8CCiJhE8x+ERaX2TwOXDXSwzFySmXMzc+6EXfZoozxJkqTRq2MzvDRLEP6uXF9Wbr8MuAIgM1dHxOo2+ljW0se5NAF0HvDFzHwO+O+I+GbZ5jCagHwfQER8jjJTuR32Br4CvDkz7ykz1scCVzcTzEAzS9vr6lJPry9n5hbgnojYt7SdVC53lttTaALwd7azxl5fLbO4a4AJwNdK+xqa5RCHAq8Abi61TwD6nd2VJEmqTUcCb3lJ/wTgyIhImoCVvBD02uljAvAW4LSIuBgIYFpE7NaBkvvzOPBfNLO099DMhv8yM2cNsP2v+9x+uuV6tPz8SGZ+chjrfP5YmbklIjZnZpb2LTTnOIC7M/O1w3xcSZKkUa9TSxoWAVdl5oGZ2ZOZBwAPAispb76KiFcARw3Sx3xgdWYeUPo4kGZ29wyaGdGzImJCREynWScMzbrgnog4uNzekTe6PVOOdW5EvC0znwAejIi3lvojIo4eYp83AheU2WIiYv+I2KeN/TYCOxL01wF7R8Rry3EnRcRv7UB/kiRJY0anljScA3y0T9u1NG/q2jki1gJraQLwYH1c108f7wXeSDODfA/NLOytAOUNbRcCN0TEk8AtlKAYEfsBK4DdgS0R8QHgiBJk+5WZv46IU2mWAvwKeDvwjxHxIWASzTKLuwYZQ9/+boqIw4Fby9KCXwG/Czy6jV1XA89FxF3AUoYwU16O+0z52LYrImIPmvP+MeDuofQjSZI0FsULr35LL7bT9ENy+nkf63YZkiRpFHro8iF9+FXHRcTKzHzRdzz4TWuSJEmqWic/paFtEXE7W3/iAcA7MnNNh487DfhGP3fNz8wNnTz2APUcCVzVp/npzHzNSNciSZJUi1EReLsV6EqondWNY/enBPxZ3a5DkiSpJi5pkCRJUtUMvJIkSaqagVeSJElVM/BKkiSpagZeSZIkVc3AK0mSpKoZeCVJklQ1A68kSZKqZuCVJElS1Qy8kiRJqpqBV5IkSVUz8EqSJKlqBl5JkiRVzcArSZKkqhl4JUmSVDUDryRJkqpm4JUkSVLVDLySJEmq2sRuF6DR7cj992DF5Qu6XYYkSdJ2c4ZXkiRJVTPwSpIkqWoGXkmSJFXNwCtJkqSqGXglSZJUNQOvJEmSqmbglSRJUtUMvJIkSaqagVeSJElVM/BKkiSpagZeSZIkVc3AK0mSpKoZeCVJklQ1A68kSZKqZuCVJElS1Qy8kiRJqpqBV5IkSVUz8EqSJKlqBl5JkiRVzcArSZKkqhl4JUmSVDUDryRJkqpm4JUkSVLVDLySJEmqmoFXkiRJVTPwSpIkqWqRmd2uQaNYRGwE1nW7ji7YC3is20V0geMef8br2B33+DNexz7exn1gZu7dt3FiNyrRmLIuM+d2u4iRFhErHPf4MV7HDeN37I57/BmvYx+v4+7LJQ2SJEmqmoFXkiRJVTPwaluWdLuALnHc48t4HTeM37E77vFnvI59vI57K75pTZIkSVVzhleSJElVM/BKkiSpagZe9SsiTo6IdRFxf0Rc1O16tkdEHBAR34qIeyLi7oh4f2m/NCIejohV5fLGln3+tIx5XUS8oaW938cjImZGxO2l/UsR8ZKRHWX/IuKhiFhTxreitE2NiJsj4r7yc8/SHhFxRRnD6oiY3dLPeWX7+yLivJb2OaX/+8u+MfKjfLGIOLTlvK6KiCci4gM1nvOI+HREPBoRP2hp6/g5HugYXR73X0XEvWVs10XES0t7T0Q81XLeP7G94xvsMRwpA4y947/bEbFTuX1/ub9nhIbcW1d/4/5Sy5gfiohVpb2acx4DP4dV/3feEZnpxctWF2AC8ABwEPAS4C7giG7XtR3jmA7MLtd3A34IHAFcCizuZ/sjylh3AmaWx2DCYI8H8C/A2eX6J4D3dnvcpZaHgL36tP0lcFG5fhHw0XL9jcBXgQCOAW4v7VOBH5Wfe5bre5b7vle2jbLvKd0e8wC/xz8FDqzxnAPzgNnAD0byHA90jC6P+yRgYrn+0ZZx97Ru16efIY1voMdwFIy947/bwP8GPlGunw18qdvj7nP/XwN/Vts5Z+DnsOr/zjtxcYZX/Xk1cH9m/igznwGWAad1uaYhy8xHMvOOcn0jsBbYf5BdTgOWZebTmfkgcD/NY9Hv41H+J3wCcE3Z/0rg9I4MZnicRlMjbF3racBns3Eb8NKImA68Abg5M3+Rmf8D3AycXO7bPTNvy+Zfw88yOsc9H3ggM388yDZj9pxn5neAX/RpHolzPNAxRkR/487MmzLz2XLzNmDGYH1s5/gGegxHzADnfCDD+bvd+phcA8zvnQkcCYONu9RxJvDFwfoYi+d8kOew6v/OO8HAq/7sD/yk5fZ6Bg+Ko155Ce6VwO2l6Q/KSz6fbnmpZqBxD9Q+DfhlyxPtaHqcErgpIlZGxIWlbd/MfKRc/ymwb7k+1HHvX673bR9tzmbrJ8HazzmMzDke6BijxQU0M1W9ZkbEnRHx7xHx26Vte8Y3mv9d7PTv9vP7lPsfL9uPBr8N/Cwz72tpq+6c93kO8+98Oxh4Vb2ImAJcC3wgM58A/hE4GJgFPELzclhtjsvM2cApwO9HxLzWO8v/5qv9TMKy9nAhcHVpGg/nfCsjcY5H2+9RRFwMPAt8vjQ9AvxmZr4S+GPgCxGxe7v9jbbxDWDc/W73cQ5b/8e2unPez3PY88bj3/n2MvCqPw8DB7TcnlHaxpyImETzD8XnM/NfATLzZ5n5XGZuAT5F8xIfDDzugdo30LxkNLFPe9dl5sPl56PAdTRj/Fnvy3Hl56Nl86GO+2G2fsl41Iy7xSnAHZn5Mxgf57wYiXM80DG6KiLOB04F3l6eoCkv528o11fSrF19Ods3vlH57+II/W4/v0+5f4+yfVeVWt4MfKm3rbZz3t9zGOP473xHGHjVn+8Dh0Tzjt2X0Lw0vLzLNQ1ZWdv1z8DazPyblvbWNVhnAL3v/F0OnB3NO5JnAofQLOjv9/EoT6rfAhaV/c8DvtLJMbUjInaNiN16r9O8oecHNOPrfXdua63LgXPLO3yPAR4vL2XdCJwUEXuWl0lPAm4s9z0REceUx/hcRsG4+9hq1qf2c95iJM7xQMfomog4GfgTYGFmPtnSvndETCjXD6I5vz/azvEN9Bh21Qj9brc+JouAb/b+p6LLXg/cm5nPvyxf0zkf6DlsO+qt4u98h+UoeOecl9F3oXm35w9p/nd8cbfr2c4xHEfzMsxqYFW5vBG4ClhT2pcD01v2ubiMeR0tnzww0ONB807n79G8IeRqYKdRMO6DaN55fRdwd2+9NGvuvgHcB3wdmFraA/j7MrY1wNyWvi4oY7sfeGdL+1yaJ9YHgI9TvrVxNFyAXWlmn/ZoaavunNME+keAzTRr7941Eud4oGN0edz306xR7P077/1EgbeUv4FVwB3Am7Z3fIM9hl0ee8d/t4HJ5fb95f6Duj3u0r4UeE+fbas55wz8HFb933knLn61sCRJkqrmkgZJkiRVzcArSZKkqhl4JUmSVDUDryRJkqpm4JUkSVLVDLySJEmqmoFXkiRJVfv/EfbESWw+va4AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "gr2 = df.loc[(df.args_op_name == 'Add') & (df.dur > 10), ['dur', \"name\"]].groupby(\"name\").sum().sort_values('dur')\n", + "gr2.plot.barh(figsize=(10, 4));" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "scrolled": false + }, + "source": [ + "## onnxruntime" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "from onnxruntime import InferenceSession, RunOptions, SessionOptions\n", + "so = SessionOptions()\n", + "so.enable_profiling = True\n", + "sess = InferenceSession(onx.SerializeToString(), so)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(0, 111):\n", + " sess.run(None, {'X': X.astype(numpy.float32)}, )" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'onnxruntime_profile__2021-03-18_23-16-39.json'" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "prof = sess.end_profiling()\n", + "prof" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'cat': 'Session',\n", + " 'pid': 504784,\n", + " 'tid': 424180,\n", + " 'dur': 353,\n", + " 'ts': 6,\n", + " 'ph': 'X',\n", + " 'name': 'model_loading_array',\n", + " 'args': {}},\n", + " {'cat': 'Session',\n", + " 'pid': 504784,\n", + " 'tid': 424180,\n", + " 'dur': 3921,\n", + " 'ts': 393,\n", + " 'ph': 'X',\n", + " 'name': 'session_initialization',\n", + " 'args': {}},\n", + " {'cat': 'Node',\n", + " 'pid': 504784,\n", + " 'tid': 424180,\n", + " 'dur': 1,\n", + " 'ts': 37730,\n", + " 'ph': 'X',\n", + " 'name': 'Re_ReduceSumSquare_fence_before',\n", + " 'args': {'op_name': 'ReduceSumSquare'}}]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with open(prof, \"r\") as f:\n", + " js = json.load(f)\n", + " \n", + "js[:3]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
catpidtiddurtsphnameargs_op_nameargs_providerargs_graph_indexargs_parameter_sizeargs_exec_plan_indexargs_activation_sizeargs_output_size
0Session5047844241803536Xmodel_loading_arrayNaNNaNNaNNaNNaNNaNNaN
1Session5047844241803921393Xsession_initializationNaNNaNNaNNaNNaNNaNNaN
2Node504784424180137730XRe_ReduceSumSquare_fence_beforeReduceSumSquareNaNNaNNaNNaNNaNNaN
3Node504784424180311837737XRe_ReduceSumSquare_kernel_timeReduceSumSquareCPUExecutionProvider0008000000400000
4Node504784424180040868XRe_ReduceSumSquare_fence_afterReduceSumSquareNaNNaNNaNNaNNaNNaN
.............................................
2550Node50478442418002038167XAr_ArgMin_fence_beforeArgMinNaNNaNNaNNaNNaNNaN
2551Node50478442418032762038168XAr_ArgMin_kernel_timeArgMinCPUExecutionProvider5053200000800000
2552Node50478442418002041449XAr_ArgMin_fence_afterArgMinNaNNaNNaNNaNNaNNaN
2553Session504784424180113862030066XSequentialExecutor::ExecuteNaNNaNNaNNaNNaNNaNNaN
2554Session504784424180114012030058Xmodel_runNaNNaNNaNNaNNaNNaNNaN
\n", + "

2555 rows \u00d7 14 columns

\n", + "
" + ], + "text/plain": [ + " cat pid tid dur ts ph \\\n", + "0 Session 504784 424180 353 6 X \n", + "1 Session 504784 424180 3921 393 X \n", + "2 Node 504784 424180 1 37730 X \n", + "3 Node 504784 424180 3118 37737 X \n", + "4 Node 504784 424180 0 40868 X \n", + "... ... ... ... ... ... .. \n", + "2550 Node 504784 424180 0 2038167 X \n", + "2551 Node 504784 424180 3276 2038168 X \n", + "2552 Node 504784 424180 0 2041449 X \n", + "2553 Session 504784 424180 11386 2030066 X \n", + "2554 Session 504784 424180 11401 2030058 X \n", + "\n", + " name args_op_name args_provider \\\n", + "0 model_loading_array NaN NaN \n", + "1 session_initialization NaN NaN \n", + "2 Re_ReduceSumSquare_fence_before ReduceSumSquare NaN \n", + "3 Re_ReduceSumSquare_kernel_time ReduceSumSquare CPUExecutionProvider \n", + "4 Re_ReduceSumSquare_fence_after ReduceSumSquare NaN \n", + "... ... ... ... \n", + "2550 Ar_ArgMin_fence_before ArgMin NaN \n", + "2551 Ar_ArgMin_kernel_time ArgMin CPUExecutionProvider \n", + "2552 Ar_ArgMin_fence_after ArgMin NaN \n", + "2553 SequentialExecutor::Execute NaN NaN \n", + "2554 model_run NaN NaN \n", + "\n", + " args_graph_index args_parameter_size args_exec_plan_index \\\n", + "0 NaN NaN NaN \n", + "1 NaN NaN NaN \n", + "2 NaN NaN NaN \n", + "3 0 0 0 \n", + "4 NaN NaN NaN \n", + "... ... ... ... \n", + "2550 NaN NaN NaN \n", + "2551 5 0 5 \n", + "2552 NaN NaN NaN \n", + "2553 NaN NaN NaN \n", + "2554 NaN NaN NaN \n", + "\n", + " args_activation_size args_output_size \n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 8000000 400000 \n", + "4 NaN NaN \n", + "... ... ... \n", + "2550 NaN NaN \n", + "2551 3200000 800000 \n", + "2552 NaN NaN \n", + "2553 NaN NaN \n", + "2554 NaN NaN \n", + "\n", + "[2555 rows x 14 columns]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pandas import DataFrame\n", + "from mlprodict.onnxrt.ops_whole.session import OnnxWholeSession\n", + "\n", + "df = DataFrame(OnnxWholeSession.process_profiling(js))\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst b/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst index 5f2b25103..8fefeaa8f 100644 --- a/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst +++ b/_doc/sphinxdoc/source/tutorial/numpy_api_onnx.rst @@ -27,7 +27,8 @@ and automatically have it converted into :epkg:`ONNX`. Available notebooks: -* :ref:`numpyapionnxrst` +* :ref:`numpyapionnxftrrst` +* :ref:`numpyapionnxcclrst` Principle +++++++++ diff --git a/_unittests/ut_documentation/test_run_notebooks_onnx_numpy.py b/_unittests/ut_documentation/test_run_notebooks_onnx_numpy.py index 8de1a7a09..bbe1f4358 100644 --- a/_unittests/ut_documentation/test_run_notebooks_onnx_numpy.py +++ b/_unittests/ut_documentation/test_run_notebooks_onnx_numpy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -@brief test log(time=15s) +@brief test log(time=30s) """ import os import unittest diff --git a/_unittests/ut_documentation/test_run_notebooks_onnx_memory.py b/_unittests/ut_documentation/test_run_notebooks_onnx_profile.py similarity index 91% rename from _unittests/ut_documentation/test_run_notebooks_onnx_memory.py rename to _unittests/ut_documentation/test_run_notebooks_onnx_profile.py index 21f2709d1..8d3c561b4 100644 --- a/_unittests/ut_documentation/test_run_notebooks_onnx_memory.py +++ b/_unittests/ut_documentation/test_run_notebooks_onnx_profile.py @@ -11,12 +11,12 @@ import mlprodict -class TestNotebookOnnxMemory(ExtTestCase): +class TestNotebookOnnxProfile(ExtTestCase): def setUp(self): add_missing_development_version(["jyquickhelper"], __file__, hide=True) - def test_notebook_onnx_mem(self): + def test_notebook_onnx_profile(self): fLOG( __file__, self._testMethodName, diff --git a/_unittests/ut_npy/test_custom_embedded_models.py b/_unittests/ut_npy/test_custom_embedded_models.py new file mode 100644 index 000000000..dbad6228f --- /dev/null +++ b/_unittests/ut_npy/test_custom_embedded_models.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +""" +@brief test log(time=3s) +""" +import unittest +from logging import getLogger +import numpy +from sklearn.base import ClassifierMixin, BaseEstimator +from sklearn.linear_model import LogisticRegression +from pyquickhelper.pycode import ExtTestCase, ignore_warnings +from mlprodict.onnx_conv import to_onnx +from mlprodict.onnxrt import OnnxInference +from mlprodict.npy import onnxsklearn_class +from mlprodict.npy.onnx_variable import MultiOnnxVar +import mlprodict.npy.numpy_onnx_impl as nxnp +import mlprodict.npy.numpy_onnx_impl_skl as nxnpskl + + +@onnxsklearn_class("onnx_graph") +class TwoLogisticRegressionOnnx(ClassifierMixin, BaseEstimator): + + def __init__(self): + ClassifierMixin.__init__(self) + BaseEstimator.__init__(self) + + def fit(self, X, y, sample_weights=None): + if sample_weights is not None: + raise NotImplementedError( + "weighted sample not implemented in this example.") + + # Barycenters + self.weights_ = numpy.array([(y == 0).sum(), (y == 1).sum()]) + p1 = X[y == 0].sum(axis=0) / self.weights_[0] + p2 = X[y == 1].sum(axis=0) / self.weights_[1] + self.centers_ = numpy.vstack([p1, p2]) # pylint: disable=W0201 + + # A vector orthogonal + v = p2 - p1 + v /= numpy.linalg.norm(v) + x = numpy.random.randn(X.shape[1]) + x -= x.dot(v) * v + x /= numpy.linalg.norm(x) + self.hyperplan_ = x.reshape((-1, 1)) # pylint: disable=W0201 + + # sign + sign = ((X - p1) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel() + + # Trains models + self.lr0_ = LogisticRegression().fit( # pylint: disable=W0201 + X[sign == 0], y[sign == 0]) + self.lr1_ = LogisticRegression().fit( # pylint: disable=W0201 + X[sign == 1], y[sign == 1]) + + return self + + def onnx_graph(self, X): + h = self.hyperplan_.astype(X.dtype) + c = self.centers_.astype(X.dtype) + + sign = ((X - c[0]) @ h) >= numpy.array([0], dtype=X.dtype) + cast = sign.astype(X.dtype).reshape((-1, 1)) + + prob0 = nxnpskl.logistic_regression(X, self.lr0_)[1] + prob1 = nxnpskl.logistic_regression(X, self.lr1_)[1] + prob = prob1 * cast - prob0 * (cast - numpy.array([1], dtype=X.dtype)) + label = nxnp.argmax(prob, axis=1) + return MultiOnnxVar(label, prob) + + +class TestCustomClassifier(ExtTestCase): + + def setUp(self): + logger = getLogger('skl2onnx') + logger.disabled = True + + @ignore_warnings((DeprecationWarning, RuntimeWarning)) + def test_function_classifier_embedded(self): + X = numpy.random.randn(20, 2).astype(numpy.float32) + y = ((X.sum(axis=1) + numpy.random.randn( + X.shape[0]).astype(numpy.float32)) >= 0).astype(numpy.int64) + dec = TwoLogisticRegressionOnnx() + dec.fit(X, y) + onx = to_onnx(dec, X.astype(numpy.float32)) + oinf = OnnxInference(onx) + exp = dec.predict(X) + prob = dec.predict_proba(X) + got = oinf.run({'X': X}) + self.assertEqualArray(exp, got['label'].ravel()) + self.assertEqualArray(prob, got['probabilities']) + + +if __name__ == "__main__": + unittest.main() diff --git a/_unittests/ut_npy/test_numpy_onnx_pyrt_skl.py b/_unittests/ut_npy/test_numpy_onnx_pyrt_skl.py new file mode 100644 index 000000000..6daf3467c --- /dev/null +++ b/_unittests/ut_npy/test_numpy_onnx_pyrt_skl.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +""" +@brief test log(time=3s) +""" +import unittest +import numpy +from pyquickhelper.pycode import ExtTestCase +from sklearn.datasets import make_classification +from sklearn.linear_model import LogisticRegression +from mlprodict.onnxrt import OnnxInference +import mlprodict.npy.numpy_onnx_pyrt_skl as nxnpyskl +from onnxruntime import __version__ as ort_version + + +class TestNumpyOnnxFunctionSkl(ExtTestCase): + + def common_test_clas(self, x, model_class, nxfct, key, dtype_out=None, ort=True, **kwargs): + X, y = make_classification( + 100, n_informative=2, n_features=2, n_redundant=0) + if not isinstance(key, tuple): + key = (key, ) + model = model_class().fit(X, y) + expected = model.predict(x), model.predict_proba(x) + got = nxfct(x, model) + self.assertIn(key, nxfct.signed_compiled) + got = nxfct[key](x) + compiled = nxfct[key].compiled + self.assertEqualArray(expected[0], got[0]) + self.assertEqualArray(expected[1], got[1]) + if ort: + onx = compiled.onnx_ + rt2 = OnnxInference(onx, runtime="onnxruntime1") + inputs = rt2.input_names + outputs = rt2.output_names + data = {inputs[0]: x} + got2 = rt2.run(data)[outputs[0]] + self.assertEqualArray(expected, got2, decimal=6) + + def test_logistic_regression_float32(self): + x = numpy.array([[-6.1, 5], [-3.5, 7.8]], dtype=numpy.float32) + self.common_test_clas(x, LogisticRegression, nxnpyskl.logistic_regression, + numpy.float32) + + +if __name__ == "__main__": + # TestNumpyOnnxFunction().test_pad_float32() + unittest.main() diff --git a/_unittests/ut_npy/test_onnx_variable.py b/_unittests/ut_npy/test_onnx_variable.py index a75517eb6..14745b51b 100644 --- a/_unittests/ut_npy/test_onnx_variable.py +++ b/_unittests/ut_npy/test_onnx_variable.py @@ -137,6 +137,13 @@ def test_abs_greater(x: NDArray[Any, numpy.float32], return nxnp.abs(x) > x +@onnxnumpy_default +def test_abs_greater_or_equal(x: NDArray[Any, numpy.float32], + ) -> NDArray[Any, numpy_bool]: + "onnx numpy greater or equal" + return nxnp.abs(x) >= x + + @onnxnumpy_default def test_abs_less(x: NDArray[Any, numpy.float32], ) -> NDArray[Any, numpy_bool]: @@ -144,6 +151,13 @@ def test_abs_less(x: NDArray[Any, numpy.float32], return nxnp.abs(x) < x +@onnxnumpy_default +def test_abs_less_or_equal(x: NDArray[Any, numpy.float32], + ) -> NDArray[Any, numpy_bool]: + "onnx numpy less or equal" + return nxnp.abs(x) <= x + + @onnxnumpy_default def test_abs_and(x: NDArray[Any, numpy.float32], ) -> NDArray[Any, numpy_bool]: @@ -458,12 +472,24 @@ def test_py_abs_greater(self): y = test_abs_greater(x) self.assertEqualArray(y, numpy.abs(x) > x) + @ignore_warnings(DeprecationWarning) + def test_py_abs_greater_or_equal(self): + x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) + y = test_abs_greater_or_equal(x) + self.assertEqualArray(y, numpy.abs(x) >= x) + @ignore_warnings(DeprecationWarning) def test_py_abs_less(self): x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) y = test_abs_less(x) self.assertEqualArray(y, numpy.abs(x) < x) + @ignore_warnings(DeprecationWarning) + def test_py_abs_less_or_equal(self): + x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) + y = test_abs_less_or_equal(x) + self.assertEqualArray(y, numpy.abs(x) <= x) + @ignore_warnings(DeprecationWarning) def test_py_abs_and(self): x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) diff --git a/_unittests/ut_npy/test_onnx_variable_ort.py b/_unittests/ut_npy/test_onnx_variable_ort.py index 2c91d5146..f82b68042 100644 --- a/_unittests/ut_npy/test_onnx_variable_ort.py +++ b/_unittests/ut_npy/test_onnx_variable_ort.py @@ -138,6 +138,13 @@ def test_abs_greater(x: NDArray[Any, numpy.float32], return nxnp.abs(x) > x +@onnxnumpy(runtime='onnxruntime1') +def test_abs_greater_or_equal(x: NDArray[Any, numpy.float32], + ) -> NDArray[Any, numpy_bool]: + "onnx numpy greater or equal" + return nxnp.abs(x) >= x + + @onnxnumpy(runtime='onnxruntime1') def test_abs_less(x: NDArray[Any, numpy.float32], ) -> NDArray[Any, numpy_bool]: @@ -145,6 +152,13 @@ def test_abs_less(x: NDArray[Any, numpy.float32], return nxnp.abs(x) < x +@onnxnumpy(runtime='onnxruntime1') +def test_abs_less_or_equal(x: NDArray[Any, numpy.float32], + ) -> NDArray[Any, numpy_bool]: + "onnx numpy less or equal" + return nxnp.abs(x) <= x + + @onnxnumpy(runtime='onnxruntime1') def test_abs_and(x: NDArray[Any, numpy.float32], ) -> NDArray[Any, numpy_bool]: @@ -438,12 +452,24 @@ def test_ort_abs_greater(self): y = test_abs_greater(x) self.assertEqualArray(y, numpy.abs(x) > x) + @ignore_warnings(DeprecationWarning) + def test_ort_abs_greater_or_equal(self): + x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) + y = test_abs_greater_or_equal(x) + self.assertEqualArray(y, numpy.abs(x) >= x) + @ignore_warnings(DeprecationWarning) def test_ort_abs_less(self): x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) y = test_abs_less(x) self.assertEqualArray(y, numpy.abs(x) < x) + @ignore_warnings(DeprecationWarning) + def test_ort_abs_less_or_equal(self): + x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) + y = test_abs_less_or_equal(x) + self.assertEqualArray(y, numpy.abs(x) <= x) + @ignore_warnings(DeprecationWarning) def test_ort_abs_and(self): x = numpy.array([[6.1, -5], [3.5, -7.8]], dtype=numpy.float32) diff --git a/mlprodict/npy/numpy_onnx_impl_skl.py b/mlprodict/npy/numpy_onnx_impl_skl.py new file mode 100644 index 000000000..54f4287c7 --- /dev/null +++ b/mlprodict/npy/numpy_onnx_impl_skl.py @@ -0,0 +1,13 @@ +""" +@file +@brief :epkg:`numpy` functions implemented with :epkg:`onnx`. + +.. versionadded:: 0.6 +""" +from skl2onnx.algebra.onnx_operator import OnnxSubOperator +from .onnx_variable import MultiOnnxVar + + +def logistic_regression(x, model=None): + "See :epkg:`sklearn:linear_model:LogisticRegression`." + return MultiOnnxVar(model, x, op=OnnxSubOperator) diff --git a/mlprodict/npy/numpy_onnx_pyrt_skl.py b/mlprodict/npy/numpy_onnx_pyrt_skl.py new file mode 100644 index 000000000..8e3f32490 --- /dev/null +++ b/mlprodict/npy/numpy_onnx_pyrt_skl.py @@ -0,0 +1,19 @@ +""" +@file +@brief :epkg:`numpy` functions implemented with :epkg:`onnx` +and compiled with this python runtime. + +.. versionadded:: 0.6 +""" +import numpy +from .onnx_numpy_annotation import NDArrayType +from .numpy_onnx_impl_skl import ( + logistic_regression as nx_logistic_regression, +) +from .onnx_numpy_wrapper import onnxnumpy_np + + +@onnxnumpy_np(signature=NDArrayType("T:all", ((numpy.int64,), "T"))) +def logistic_regression(x, model=None): + "logistic_regression" + return nx_logistic_regression(model, x) diff --git a/mlprodict/npy/onnx_numpy_wrapper.py b/mlprodict/npy/onnx_numpy_wrapper.py index 09b86eb64..cf6a5c234 100644 --- a/mlprodict/npy/onnx_numpy_wrapper.py +++ b/mlprodict/npy/onnx_numpy_wrapper.py @@ -175,7 +175,7 @@ def __call__(self, *args, **kwargs): tensor in *args* defines the templated version of the function to convert into *ONNX*. """ - key = tuple(a if a is None # or not hasattr(a, 'dtype')) + key = tuple(a if (a is None or hasattr(a, 'fit')) else a.dtype.type for a in args) if len(self.kwargs) == 0: return self[key](*args) diff --git a/mlprodict/npy/onnx_variable.py b/mlprodict/npy/onnx_variable.py index b3ac46539..62aeaac3f 100644 --- a/mlprodict/npy/onnx_variable.py +++ b/mlprodict/npy/onnx_variable.py @@ -14,9 +14,9 @@ OnnxDiv, OnnxEqual, OnnxFlatten, - OnnxGather, OnnxGreater, + OnnxGather, OnnxGreater, OnnxGreaterOrEqual, OnnxIdentity, - OnnxLess, + OnnxLess, OnnxLessOrEqual, OnnxMatMul, OnnxMod, OnnxMul, OnnxNeg, OnnxNot, OnnxOr, @@ -97,6 +97,9 @@ def _guess_dtype(self, dtype): dtypes.append(numpy.int64) elif isinstance(inp, float): dtypes.append(numpy.float64) + elif hasattr(inp, 'fit'): + # scikit-learn model + continue else: raise TypeError( "Unexpected type for input %i type=%r." % (i, type(inp))) @@ -149,7 +152,10 @@ def to_algebra(self, op_version=None): new_inputs = [] for inp in self.inputs: - if isinstance(inp, ( + if hasattr(inp, 'fit'): + # scikit-learn model + new_inputs.append(inp) + elif isinstance(inp, ( int, float, str, numpy.ndarray, numpy.int32, numpy.int64, numpy.float32, numpy.float64, numpy_bool, numpy_str, numpy.int8, numpy.uint8, @@ -262,10 +268,18 @@ def __ne__(self, y): "Difference." return OnnxVar(OnnxVar(self, y, op=OnnxEqual), op=OnnxNot) + def __ge__(self, y): + "Greater or Equal." + return OnnxVar(self, y, op=OnnxGreaterOrEqual) + def __gt__(self, y): "Greater." return OnnxVar(self, y, op=OnnxGreater) + def __le__(self, y): + "Less or Equal." + return OnnxVar(self, y, op=OnnxLessOrEqual) + def __lt__(self, y): "Less." return OnnxVar(self, y, op=OnnxLess) diff --git a/mlprodict/onnxrt/ops_whole/session.py b/mlprodict/onnxrt/ops_whole/session.py index d2f70ed3b..4fd48953b 100644 --- a/mlprodict/onnxrt/ops_whole/session.py +++ b/mlprodict/onnxrt/ops_whole/session.py @@ -71,14 +71,14 @@ def run(self, inputs): """ return self.sess.run(None, inputs, self.run_options) - def get_profiling(self): + @staticmethod + def process_profiling(js): """ - Returns the profiling informations. + Flattens json returned by onnxruntime profiling. + + :param js: json + :return: list of dictionaries """ - prof = self.sess.end_profiling() - with open(prof, 'r') as f: - content = f.read() - js = json.loads(content) rows = [] for row in js: if 'args' in row and isinstance(row['args'], dict): @@ -87,3 +87,13 @@ def get_profiling(self): del row['args'] rows.append(row) return rows + + def get_profiling(self): + """ + Returns the profiling informations. + """ + prof = self.sess.end_profiling() + with open(prof, 'r') as f: + content = f.read() + js = json.loads(content) + return OnnxWholeSession.process_profiling(js) diff --git a/mlprodict/tools/onnx_manipulations.py b/mlprodict/tools/onnx_manipulations.py index 3bddb62f8..5d4348394 100644 --- a/mlprodict/tools/onnx_manipulations.py +++ b/mlprodict/tools/onnx_manipulations.py @@ -27,21 +27,46 @@ def enumerate_model_node_outputs(model, add_node=False): def select_model_inputs_outputs(model, outputs=None, inputs=None, - infer_shapes=False, overwrite=None): + infer_shapes=False, overwrite=None, + verbose=0, fLOG=None): """ Takes a model and changes its outputs. - @param model :epkg:`ONNX` model - @param inputs new inputs, same ones if None - @param outputs new outputs, same ones if None - @param infer_shapes infer inputs and outputs shapes - @param overwrite overwrite type and shapes for - inputs or outputs, *overwrite* is a - dictionary `{'name': (numpy dtype, shape)}` - @return modified model + :param model: :epkg:`ONNX` model + :param inputs: new inputs, same ones if None + :param outputs: new outputs, same ones if None + :param infer_shapes: infer inputs and outputs shapes + :param overwrite: overwrite type and shapes for + inputs or outputs, *overwrite* is a + dictionary `{'name': (numpy dtype, shape)}` + :param verbose: display information while converting + :param fLOG: logging function + :return: modified model The function removes unneeded nodes. + .. exref:: + :title: Change ONNX model inputs + + The following exampels shows how to change the inputs of model + to bypass the first nodes. Shape inferences fails to determine + the new inputs type. They need to be overwritten. + `verbose=1, fLOG=print` shows the number of deleted nodes. + + :: + + import onnx + from mlprodict.tools.onnx_manipulations import select_model_inputs_outputs + + onx = onnx.load(path) + onx2 = select_model_inputs_outputs( + onx, inputs=["SentenceTokenizer/SentencepieceTokenizeOp:0", + "SentenceTokenizer/SentencepieceTokenizeOp:1"], + infer_shapes=True, verbose=1, fLOG=print, + overwrite={'SentenceTokenizer/SentencepieceTokenizeOp:0': (numpy.int32, None), + 'SentenceTokenizer/SentencepieceTokenizeOp:1': (numpy.int64, None)}) + onnx.save(onx2, path2) + .. versionchanged:: 0.6 Supports the case where inputs are changed. """ @@ -159,6 +184,12 @@ def select_model_inputs_outputs(model, outputs=None, inputs=None, value_info.name = name var_out.append(value_info) + if verbose > 0 and fLOG is not None: + fLOG("[select_model_inputs_outputs] nodes %r --> %r" % ( + len(model.graph.node), len(keep_nodes))) + fLOG("[select_model_inputs_outputs] inputs: %r" % var_in) + fLOG("[select_model_inputs_outputs] inputs: %r" % var_out) + graph = helper.make_graph(keep_nodes, model.graph.name, var_in, var_out, model.graph.initializer) onnx_model = helper.make_model(graph) From f23ff7f8f7a0398ff76921efafa181a4afc4286b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 25 Mar 2021 00:44:28 +0100 Subject: [PATCH 03/10] update for embedded models --- _doc/sphinxdoc/source/api/npy.rst | 13 +++-- .../ut_npy/test_custom_embedded_models.py | 13 +++-- _unittests/ut_npy/test_custom_transformer.py | 4 +- _unittests/ut_npy/test_numpy_onnx_pyrt.py | 5 +- _unittests/ut_npy/test_numpy_onnx_pyrt_skl.py | 17 +++++-- mlprodict/npy/numpy_onnx_impl.py | 2 +- mlprodict/npy/numpy_onnx_impl_skl.py | 6 +-- mlprodict/npy/numpy_onnx_pyrt_skl.py | 6 +-- mlprodict/npy/onnx_numpy_compiler.py | 3 +- mlprodict/npy/onnx_variable.py | 48 +++++++++++++++++-- mlprodict/onnxrt/ops_whole/session.py | 2 +- 11 files changed, 89 insertions(+), 30 deletions(-) diff --git a/_doc/sphinxdoc/source/api/npy.rst b/_doc/sphinxdoc/source/api/npy.rst index e3d6ef02f..e5ff15589 100644 --- a/_doc/sphinxdoc/source/api/npy.rst +++ b/_doc/sphinxdoc/source/api/npy.rst @@ -115,10 +115,13 @@ Registration .. _l-numpy-onnxpy-list-fct: -Available numpy functions implemented with ONNX operators -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +Available functions implemented with ONNX operators ++++++++++++++++++++++++++++++++++++++++++++++++++++ -All functions are implemented in submodule :ref:`f-numpyonnximpl`. +All functions are implemented in two submodules: + +* *numpy function*: :ref:`f-numpyonnximpl` +* *machine learned models:* :ref:`f-numpyonnximplskl` ONNX functions executed python ONNX runtime +++++++++++++++++++++++++++++++++++++++++++ @@ -129,4 +132,6 @@ Same function as above, the import goes from These function are usually not used except in unit test or as reference for more complex functions. See the source on github, `numpy_onnx_pyrt.py `_. +blob/master/mlprodict/npy/numpy_onnx_pyrt.py>`_ and +`numpy_onnx_pyrt_skl.py `_. diff --git a/_unittests/ut_npy/test_custom_embedded_models.py b/_unittests/ut_npy/test_custom_embedded_models.py index dbad6228f..ff3c02d46 100644 --- a/_unittests/ut_npy/test_custom_embedded_models.py +++ b/_unittests/ut_npy/test_custom_embedded_models.py @@ -29,7 +29,8 @@ def fit(self, X, y, sample_weights=None): "weighted sample not implemented in this example.") # Barycenters - self.weights_ = numpy.array([(y == 0).sum(), (y == 1).sum()]) + self.weights_ = numpy.array( # pylint: disable=W0201 + [(y == 0).sum(), (y == 1).sum()]) p1 = X[y == 0].sum(axis=0) / self.weights_[0] p2 = X[y == 1].sum(axis=0) / self.weights_[1] self.centers_ = numpy.vstack([p1, p2]) # pylint: disable=W0201 @@ -60,8 +61,10 @@ def onnx_graph(self, X): sign = ((X - c[0]) @ h) >= numpy.array([0], dtype=X.dtype) cast = sign.astype(X.dtype).reshape((-1, 1)) - prob0 = nxnpskl.logistic_regression(X, self.lr0_)[1] - prob1 = nxnpskl.logistic_regression(X, self.lr1_)[1] + prob0 = nxnpskl.logistic_regression( # pylint: disable=E1136 + X, model=self.lr0_)[1] + prob1 = nxnpskl.logistic_regression( # pylint: disable=E1136 + X, model=self.lr1_)[1] prob = prob1 * cast - prob0 * (cast - numpy.array([1], dtype=X.dtype)) label = nxnp.argmax(prob, axis=1) return MultiOnnxVar(label, prob) @@ -82,8 +85,8 @@ def test_function_classifier_embedded(self): dec.fit(X, y) onx = to_onnx(dec, X.astype(numpy.float32)) oinf = OnnxInference(onx) - exp = dec.predict(X) - prob = dec.predict_proba(X) + exp = dec.predict(X) # pylint: disable=E1101 + prob = dec.predict_proba(X) # pylint: disable=E1101 got = oinf.run({'X': X}) self.assertEqualArray(exp, got['label'].ravel()) self.assertEqualArray(prob, got['probabilities']) diff --git a/_unittests/ut_npy/test_custom_transformer.py b/_unittests/ut_npy/test_custom_transformer.py index 7fe715ef9..3acf7e2f3 100644 --- a/_unittests/ut_npy/test_custom_transformer.py +++ b/_unittests/ut_npy/test_custom_transformer.py @@ -14,7 +14,7 @@ from skl2onnx import update_registered_converter from skl2onnx.algebra.onnx_ops import ( # pylint: disable=E0611 OnnxIdentity, OnnxMatMul, OnnxSub) -from skl2onnx.algebra.onnx_operator import OnnxSubOperator +from skl2onnx.algebra.onnx_operator import OnnxSubEstimator from skl2onnx.common.data_types import guess_numpy_type from mlprodict.onnx_conv import to_onnx from mlprodict.onnxrt import OnnxInference @@ -50,7 +50,7 @@ def decorrelate_transformer_converter(scope, operator, container): opv = container.target_opset out = operator.outputs X = operator.inputs[0] - subop = OnnxSubOperator(op.pca_, X, op_version=opv) + subop = OnnxSubEstimator(op.pca_, X, op_version=opv) Y = OnnxIdentity(subop, op_version=opv, output_names=out[:1]) Y.add_to(scope, container) diff --git a/_unittests/ut_npy/test_numpy_onnx_pyrt.py b/_unittests/ut_npy/test_numpy_onnx_pyrt.py index 3e86a80ea..759c7ad3c 100644 --- a/_unittests/ut_npy/test_numpy_onnx_pyrt.py +++ b/_unittests/ut_npy/test_numpy_onnx_pyrt.py @@ -5,7 +5,7 @@ import unittest import numpy import scipy.special as sp -from pyquickhelper.pycode import ExtTestCase +from pyquickhelper.pycode import ExtTestCase, ignore_warnings from pyquickhelper.texthelper import compare_module_version from mlprodict.onnxrt import OnnxInference from mlprodict.onnxrt.ops_cpu.op_pad import onnx_pad @@ -66,6 +66,7 @@ def test_acos_float32(self): x = numpy.array([[0.5, 0.1], [-0.5, -0.1]], dtype=numpy.float32) self.common_test1(x, numpy.arccos, nxnpy.acos, numpy.float32) + @ignore_warnings(RuntimeWarning) def test_acosh_float32(self): x = numpy.array([[0.5, 0.1], [-0.5, -0.1]], dtype=numpy.float32) self.common_test1(x, numpy.arccosh, nxnpy.acosh, numpy.float32) @@ -382,5 +383,5 @@ def test_vstack_float32(self): if __name__ == "__main__": - # TestNumpyOnnxFunction().test_pad_float32() + # TestNumpyOnnxFunction().test_arange_float32() unittest.main() diff --git a/_unittests/ut_npy/test_numpy_onnx_pyrt_skl.py b/_unittests/ut_npy/test_numpy_onnx_pyrt_skl.py index 6daf3467c..3a0f43a7b 100644 --- a/_unittests/ut_npy/test_numpy_onnx_pyrt_skl.py +++ b/_unittests/ut_npy/test_numpy_onnx_pyrt_skl.py @@ -14,14 +14,16 @@ class TestNumpyOnnxFunctionSkl(ExtTestCase): - def common_test_clas(self, x, model_class, nxfct, key, dtype_out=None, ort=True, **kwargs): + def common_test_clas(self, x, model_class, nxfct, key, dtype_out=None, + ort=True, **kwargs): X, y = make_classification( 100, n_informative=2, n_features=2, n_redundant=0) if not isinstance(key, tuple): key = (key, ) model = model_class().fit(X, y) + key = key + (model, ) expected = model.predict(x), model.predict_proba(x) - got = nxfct(x, model) + got = nxfct(x, model=model) self.assertIn(key, nxfct.signed_compiled) got = nxfct[key](x) compiled = nxfct[key].compiled @@ -33,15 +35,20 @@ def common_test_clas(self, x, model_class, nxfct, key, dtype_out=None, ort=True, inputs = rt2.input_names outputs = rt2.output_names data = {inputs[0]: x} - got2 = rt2.run(data)[outputs[0]] - self.assertEqualArray(expected, got2, decimal=6) + got2 = rt2.run(data) + self.assertEqualArray(expected[0], got2[outputs[0]], decimal=6) + self.assertEqualArray(expected[1], got2[outputs[1]], decimal=6) def test_logistic_regression_float32(self): x = numpy.array([[-6.1, 5], [-3.5, 7.8]], dtype=numpy.float32) self.common_test_clas(x, LogisticRegression, nxnpyskl.logistic_regression, numpy.float32) + def test_logistic_regression_float64(self): + x = numpy.array([[-6.1, 5], [-3.5, 7.9]], dtype=numpy.float64) + self.common_test_clas(x, LogisticRegression, nxnpyskl.logistic_regression, + numpy.float64) + if __name__ == "__main__": - # TestNumpyOnnxFunction().test_pad_float32() unittest.main() diff --git a/mlprodict/npy/numpy_onnx_impl.py b/mlprodict/npy/numpy_onnx_impl.py index 2c71f12ab..bee78812d 100644 --- a/mlprodict/npy/numpy_onnx_impl.py +++ b/mlprodict/npy/numpy_onnx_impl.py @@ -97,7 +97,7 @@ def arange(start, stop, step=1): cs = OnnxVar(cst, numpy.array([0], dtype=numpy.int64), op=OnnxCumSum) - diff = start - numpy.int64(step) + diff = start - numpy.array([step], dtype=numpy.int64) return OnnxVar(cs, diff, op=OnnxAdd) diff --git a/mlprodict/npy/numpy_onnx_impl_skl.py b/mlprodict/npy/numpy_onnx_impl_skl.py index 54f4287c7..0d6710e6e 100644 --- a/mlprodict/npy/numpy_onnx_impl_skl.py +++ b/mlprodict/npy/numpy_onnx_impl_skl.py @@ -4,10 +4,10 @@ .. versionadded:: 0.6 """ -from skl2onnx.algebra.onnx_operator import OnnxSubOperator +from skl2onnx.algebra.onnx_operator import OnnxSubEstimator from .onnx_variable import MultiOnnxVar -def logistic_regression(x, model=None): +def logistic_regression(x, *, model=None): "See :epkg:`sklearn:linear_model:LogisticRegression`." - return MultiOnnxVar(model, x, op=OnnxSubOperator) + return MultiOnnxVar(model, x, op=OnnxSubEstimator) diff --git a/mlprodict/npy/numpy_onnx_pyrt_skl.py b/mlprodict/npy/numpy_onnx_pyrt_skl.py index 8e3f32490..09fead369 100644 --- a/mlprodict/npy/numpy_onnx_pyrt_skl.py +++ b/mlprodict/npy/numpy_onnx_pyrt_skl.py @@ -13,7 +13,7 @@ from .onnx_numpy_wrapper import onnxnumpy_np -@onnxnumpy_np(signature=NDArrayType("T:all", ((numpy.int64,), "T"))) -def logistic_regression(x, model=None): +@onnxnumpy_np(signature=NDArrayType(("T:all", ), dtypes_out=((numpy.int64,), "T"))) +def logistic_regression(x, *, model=None): "logistic_regression" - return nx_logistic_regression(model, x) + return nx_logistic_regression(x, model=model) diff --git a/mlprodict/npy/onnx_numpy_compiler.py b/mlprodict/npy/onnx_numpy_compiler.py index 8d14d5509..fb1672361 100644 --- a/mlprodict/npy/onnx_numpy_compiler.py +++ b/mlprodict/npy/onnx_numpy_compiler.py @@ -206,9 +206,10 @@ def _parse_annotation(self, signature, version): if (signature is not None and not signature.n_variables and nv > len(kwargs)): raise RuntimeError( # pragma: no cover - "Mismatch between version=%r and kwargs=%r for " + "Mismatch (%d - %d - %d ? %d) between version=%r and kwargs=%r for " "function %r, optional argument is %d, " "signature=%r." % ( + len(version), len(args), n_opt, len(kwargs), version, kwargs, self.fct_, signature.n_variables, signature)) vvers = version[-len(kwargs):] diff --git a/mlprodict/npy/onnx_variable.py b/mlprodict/npy/onnx_variable.py index 62aeaac3f..c6572cdd7 100644 --- a/mlprodict/npy/onnx_variable.py +++ b/mlprodict/npy/onnx_variable.py @@ -26,6 +26,8 @@ OnnxSqueeze, OnnxSub, OnnxTopK, OnnxTranspose ) +from skl2onnx.algebra.onnx_operator import OnnxOperatorItem +from skl2onnx.common.data_types import _guess_numpy_type from ..tools.onnx2py_helper import guess_proto_dtype @@ -86,6 +88,8 @@ def _guess_dtype(self, dtype): dtypes.append(dt) elif isinstance(inp, OnnxVar): dtypes.append(inp.dtype) + elif isinstance(inp, MultiOnnxVar): + dtypes.append(inp._guess_dtype(dtype)) elif isinstance(inp, (numpy.float32, numpy.float64, numpy.int32, numpy.int64)): dtypes.append(inp.dtype) @@ -138,7 +142,11 @@ def to_algebra(self, op_version=None): raise RuntimeError( # pragma: no cover "Unexpected number of inputs, 1 expected, " "got {} instead.".format(self.inputs)) - self.alg_ = self.inputs[0] + if self.dtype is None or hasattr(self.inputs[0], 'onnx_name'): + self.alg_ = self.inputs[0] + else: + self.alg_ = ( + self.inputs[0], _guess_numpy_type(self.dtype, None)) else: if isinstance(self.onnx_op, str): var = self._custom_op(*self.inputs, op_version=op_version, @@ -470,6 +478,10 @@ def __init__(self, first, *args): else: self.values = None self.unique = first + if self.values is not None and self.unique is not None: + raise RuntimeError( + "Unexpected configuration. One member (values or unique) must be " + "null, unique=%r, values=%r" % (self.unique, self.values)) def __len__(self): "usual" @@ -505,11 +517,25 @@ def output_names(self): @output_names.setter def output_names(self, value): - "Updates 'output_names' of attribute 'unique'." + """ + Updates 'output_names' of attribute 'unique' + or every output name of attribute 'values'. + """ if self.values is None: - if hasattr(self.unique, 'to_onnx'): + if (hasattr(self.unique, 'to_onnx') or + hasattr(self.unique, 'add_to')): + if len(value) > 1: + self.values = tuple( + OnnxIdentity(self.unique[i], output_names=value[i:i + 1], + op_version=self.unique.op_version) + for i in range(0, len(value))) + self.unique = None + return self.unique.output_names = value return + raise NotImplementedError( + "Not implemented yet, value=%r, unique=%r values=%r." % ( + value, self.unique, self.values)) if self.values is not None and len(self.values) == len(value): for name, v in zip(value, self.values): v.output_names = [name] @@ -523,6 +549,9 @@ def to_onnx(self, *args, **kwargs): if self.values is None: if hasattr(self.unique, 'to_onnx'): return self.unique.to_onnx(*args, **kwargs) + raise NotImplementedError( + "Not implemented yet unique=%r values=%r args=%r " + "kwargs=%r." % (self.unique, self.values, args, kwargs)) if self.values is not None: if len(self.values) == len(kwargs.get('outputs', [])): return self.values[0].to_onnx( @@ -543,6 +572,10 @@ def __init__(self, *inputs, op=None, dtype=None, **kwargs): self.onxvar = OnnxVar(*inputs, op=op, dtype=None, **kwargs) self.alg_ = None + def _guess_dtype(self, dtype): + "Guesses dtype when not specified." + return self.onxvar._guess_dtype(dtype) + @property def inputs(self): "Returns `self.onxvar.inputs`." @@ -571,6 +604,9 @@ def to_algebra(self, op_version=None): numpy_bool, numpy_str, numpy.int8, numpy.uint8, numpy.int16, numpy.uint16, numpy.uint32, numpy.uint64)): new_inputs.append(inp) + elif hasattr(inp, 'fit'): + # scikit-learn models + new_inputs.append(inp) else: new_inputs.append( inp.to_algebra(op_version=op_version)) @@ -585,3 +621,9 @@ def to_algebra(self, op_version=None): *new_inputs, op_version=op_version, **self.onnx_op_kwargs) self.alg_ = TupleOnnxAny(res) return self.alg_ + + def __getitem__(self, index): + """ + Returns the ith elements. + """ + return OnnxVar(self, index=index, op=OnnxOperatorItem) diff --git a/mlprodict/onnxrt/ops_whole/session.py b/mlprodict/onnxrt/ops_whole/session.py index 4fd48953b..0c7485d7a 100644 --- a/mlprodict/onnxrt/ops_whole/session.py +++ b/mlprodict/onnxrt/ops_whole/session.py @@ -75,7 +75,7 @@ def run(self, inputs): def process_profiling(js): """ Flattens json returned by onnxruntime profiling. - + :param js: json :return: list of dictionaries """ From bc23bd8f3c7edd8044242cf3aefdcecf72c155af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 25 Mar 2021 10:34:25 +0100 Subject: [PATCH 04/10] Fix skl mechanism --- .../ut_npy/test_custom_embedded_models.py | 16 +++- mlprodict/npy/numpy_onnx_impl_skl.py | 3 +- mlprodict/npy/onnx_variable.py | 75 ++++++++++++++++++- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/_unittests/ut_npy/test_custom_embedded_models.py b/_unittests/ut_npy/test_custom_embedded_models.py index ff3c02d46..57ee2cfeb 100644 --- a/_unittests/ut_npy/test_custom_embedded_models.py +++ b/_unittests/ut_npy/test_custom_embedded_models.py @@ -76,21 +76,29 @@ def setUp(self): logger = getLogger('skl2onnx') logger.disabled = True - @ignore_warnings((DeprecationWarning, RuntimeWarning)) - def test_function_classifier_embedded(self): - X = numpy.random.randn(20, 2).astype(numpy.float32) + def common_test_function_classifier_embedded(self, dtype): + X = numpy.random.randn(20, 2).astype(dtype) y = ((X.sum(axis=1) + numpy.random.randn( X.shape[0]).astype(numpy.float32)) >= 0).astype(numpy.int64) dec = TwoLogisticRegressionOnnx() dec.fit(X, y) - onx = to_onnx(dec, X.astype(numpy.float32)) + onx = to_onnx(dec, X.astype(dtype)) oinf = OnnxInference(onx) exp = dec.predict(X) # pylint: disable=E1101 prob = dec.predict_proba(X) # pylint: disable=E1101 got = oinf.run({'X': X}) + self.assertEqual(dtype, prob.dtype) self.assertEqualArray(exp, got['label'].ravel()) self.assertEqualArray(prob, got['probabilities']) + @ignore_warnings((DeprecationWarning, RuntimeWarning)) + def test_function_classifier_embedded_float32(self): + self.common_test_function_classifier_embedded(numpy.float32) + + @ignore_warnings((DeprecationWarning, RuntimeWarning)) + def test_function_classifier_embedded_float64(self): + self.common_test_function_classifier_embedded(numpy.float64) + if __name__ == "__main__": unittest.main() diff --git a/mlprodict/npy/numpy_onnx_impl_skl.py b/mlprodict/npy/numpy_onnx_impl_skl.py index 0d6710e6e..a7b08f231 100644 --- a/mlprodict/npy/numpy_onnx_impl_skl.py +++ b/mlprodict/npy/numpy_onnx_impl_skl.py @@ -10,4 +10,5 @@ def logistic_regression(x, *, model=None): "See :epkg:`sklearn:linear_model:LogisticRegression`." - return MultiOnnxVar(model, x, op=OnnxSubEstimator) + return MultiOnnxVar(model, x, op=OnnxSubEstimator, + options={'zipmap': False}) diff --git a/mlprodict/npy/onnx_variable.py b/mlprodict/npy/onnx_variable.py index c6572cdd7..dc817b26d 100644 --- a/mlprodict/npy/onnx_variable.py +++ b/mlprodict/npy/onnx_variable.py @@ -26,7 +26,7 @@ OnnxSqueeze, OnnxSub, OnnxTopK, OnnxTranspose ) -from skl2onnx.algebra.onnx_operator import OnnxOperatorItem +from skl2onnx.algebra.onnx_operator import OnnxOperatorItem, OnnxOperator from skl2onnx.common.data_types import _guess_numpy_type from ..tools.onnx2py_helper import guess_proto_dtype @@ -236,68 +236,92 @@ def reshape(self, shape): shape = numpy.array(shape, dtype=numpy.int64) return OnnxVar(self, shape, op=OnnxReshape) + def _make_array(self, y): + """Converts *y* into an array if not.""" + if hasattr(y, 'dtype') and not isinstance(y, (numpy.ndarray, OnnxVar)): + return numpy.full((1, ), y, dtype=y.dtype) + if isinstance(y, (float, int, str)): + return numpy.array([y]) + return y + def __add__(self, y): "Addition." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxAdd) def __sub__(self, y): "Subtraction." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxSub) def __mul__(self, y): "Multiplication." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxMul) def __pow__(self, y): "Power." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxPow) def __mod__(self, y): "Modulo." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxMod) def __matmul__(self, y): "Matrix multiplication." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxMatMul) def __truediv__(self, y): "Division, no difference between `/` and `//`." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxDiv) def __floordiv__(self, y): "Division, no difference between `/` and `//`." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxDiv) def __eq__(self, y): "Equality." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxEqual) def __ne__(self, y): "Difference." + y = self._make_array(y) return OnnxVar(OnnxVar(self, y, op=OnnxEqual), op=OnnxNot) def __ge__(self, y): "Greater or Equal." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxGreaterOrEqual) def __gt__(self, y): "Greater." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxGreater) def __le__(self, y): "Less or Equal." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxLessOrEqual) def __lt__(self, y): "Less." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxLess) def __and__(self, y): "And." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxAnd) def __or__(self, y): "And." + y = self._make_array(y) return OnnxVar(self, y, op=OnnxOr) def not_(self): @@ -462,7 +486,7 @@ def flatten(self, axis=0): return fl -class TupleOnnxAny: +class TupleOnnxAny(OnnxOperator): """ Class used to return multiple @see cl OnnxVar at the same time. @@ -482,6 +506,10 @@ def __init__(self, first, *args): raise RuntimeError( "Unexpected configuration. One member (values or unique) must be " "null, unique=%r, values=%r" % (self.unique, self.values)) + if self.values is None and self.unique is None: + raise RuntimeError( + "Unexpected configuration. One member (values or unique) must be " + "not null.") def __len__(self): "usual" @@ -515,6 +543,27 @@ def output_names(self): "Not implemented yet unique=%r values=%r." % ( self.unique, self.values)) + def get_output_type_inference(self, input_shapes=None): + """ + Returns the expected output types in a list. + """ + if self.values is None: + if hasattr(self.unique, 'get_output_type_inference'): + return self.unique.get_output_type_inference(input_shapes) + raise NotImplementedError( + "Not implemented yet unique=%r values=%r." % ( + self.unique, self.values)) + + @property + def outputs(self): + "Returns 'output_names' of attribute 'unique'." + if self.values is None: + if hasattr(self.unique, 'to_onnx'): + return self.unique.outputs + raise NotImplementedError( + "Not implemented yet unique=%r values=%r." % ( + self.unique, self.values)) + @output_names.setter def output_names(self, value): """ @@ -544,6 +593,28 @@ def output_names(self, value): "Not implemented yet, value=%r, unique=%r values=%r." % ( value, self.unique, self.values)) + def add_to(self, scope, container, operator=None, run_converters=False): + """ + Adds outputs to the container if not already added, + registered the outputs if the node is not final. + + :param scope: scope + :param container: container + :param operator: overwrite inputs + :param run_converters: must be True if called from method `to_onnx` + """ + if self.values is not None: + for v in self.values: + v.add_to(scope, container, operator=operator, + run_converters=run_converters) + return + if self.unique is not None: + self.unique.add_to(scope, container, operator=operator, + run_converters=run_converters) + return + raise RuntimeError( + "Attributes 'unique' and 'values' cannot be both null.") + def to_onnx(self, *args, **kwargs): "Converts the underlying class into an ONNX graph." if self.values is None: From 9abb852d6efa96cff4ef62cb9af9b9e3401124bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 25 Mar 2021 15:35:19 +0100 Subject: [PATCH 05/10] add linear regression --- .../ut_npy/test_custom_embedded_models.py | 83 ++++++++++++++++++- mlprodict/npy/numpy_onnx_impl_skl.py | 7 +- mlprodict/npy/onnx_variable.py | 26 +++--- 3 files changed, 99 insertions(+), 17 deletions(-) diff --git a/_unittests/ut_npy/test_custom_embedded_models.py b/_unittests/ut_npy/test_custom_embedded_models.py index 57ee2cfeb..8e1cb8177 100644 --- a/_unittests/ut_npy/test_custom_embedded_models.py +++ b/_unittests/ut_npy/test_custom_embedded_models.py @@ -5,8 +5,9 @@ import unittest from logging import getLogger import numpy -from sklearn.base import ClassifierMixin, BaseEstimator -from sklearn.linear_model import LogisticRegression +from sklearn.base import ClassifierMixin, RegressorMixin, BaseEstimator +from sklearn.linear_model import LogisticRegression, LinearRegression +from sklearn.cluster import KMeans from pyquickhelper.pycode import ExtTestCase, ignore_warnings from mlprodict.onnx_conv import to_onnx from mlprodict.onnxrt import OnnxInference @@ -70,7 +71,62 @@ def onnx_graph(self, X): return MultiOnnxVar(label, prob) -class TestCustomClassifier(ExtTestCase): +@onnxsklearn_class("onnx_graph") +class TwoLinearRegressionOnnx(RegressorMixin, BaseEstimator): + + def __init__(self): + RegressorMixin.__init__(self) + BaseEstimator.__init__(self) + + def fit(self, X, y, sample_weights=None): + if sample_weights is not None: + raise NotImplementedError( + "weighted sample not implemented in this example.") + + # Barycenters + km = KMeans(n_clusters=2).fit(X) + cl = km.predict(X) + self.weights_ = numpy.array( # pylint: disable=W0201 + [(cl == 0).sum(), (cl == 1).sum()]) + self.centers_ = km.cluster_centers_.T # pylint: disable=W0201 + + # A vector orthogonal + p1 = self.centers_[0, :] + p2 = self.centers_[1, :] + v = p2 - p1 + v /= numpy.linalg.norm(v) + x = numpy.random.randn(X.shape[1]) + x -= x.dot(v) * v + x /= numpy.linalg.norm(x) + self.hyperplan_ = x.reshape((-1, 1)) # pylint: disable=W0201 + + # sign + sign = ((X - p1) @ self.hyperplan_ >= 0).astype(numpy.int64).ravel() + + # Trains models + self.lr0_ = LinearRegression().fit( # pylint: disable=W0201 + X[sign == 0], y[sign == 0]) + self.lr1_ = LinearRegression().fit( # pylint: disable=W0201 + X[sign == 1], y[sign == 1]) + + return self + + def onnx_graph(self, X): + h = self.hyperplan_.astype(X.dtype) + c = self.centers_.astype(X.dtype) + + sign = ((X - c[0]) @ h) >= numpy.array([0], dtype=X.dtype) + cast = sign.astype(X.dtype).reshape((-1, 1)) + + prob0 = nxnpskl.linear_regression( # pylint: disable=E1136 + X, model=self.lr0_)[1] + prob1 = nxnpskl.linear_regression( # pylint: disable=E1136 + X, model=self.lr1_)[1] + pred = prob1 * cast - prob0 * (cast - numpy.array([1], dtype=X.dtype)) + return pred + + +class TestCustomEmbeddedModels(ExtTestCase): def setUp(self): logger = getLogger('skl2onnx') @@ -99,6 +155,27 @@ def test_function_classifier_embedded_float32(self): def test_function_classifier_embedded_float64(self): self.common_test_function_classifier_embedded(numpy.float64) + def common_test_function_regressor_embedded(self, dtype): + X = numpy.random.randn(20, 2).astype(dtype) + y = (X.sum(axis=1) + numpy.random.randn( + X.shape[0])).astype(numpy.float32) + dec = TwoLinearRegressionOnnx() + dec.fit(X, y) + onx = to_onnx(dec, X.astype(dtype)) + oinf = OnnxInference(onx) + exp = dec.predict(X) # pylint: disable=E1101 + got = oinf.run({'X': X}) + self.assertEqual(dtype, exp.dtype) + self.assertEqualArray(exp, got['variable']) + + @ignore_warnings((DeprecationWarning, RuntimeWarning)) + def test_function_regressor_embedded_float32(self): + self.common_test_function_regressor_embedded(numpy.float32) + + @ignore_warnings((DeprecationWarning, RuntimeWarning)) + def test_function_regressor_embedded_float64(self): + self.common_test_function_regressor_embedded(numpy.float64) + if __name__ == "__main__": unittest.main() diff --git a/mlprodict/npy/numpy_onnx_impl_skl.py b/mlprodict/npy/numpy_onnx_impl_skl.py index a7b08f231..b77be30c7 100644 --- a/mlprodict/npy/numpy_onnx_impl_skl.py +++ b/mlprodict/npy/numpy_onnx_impl_skl.py @@ -5,7 +5,12 @@ .. versionadded:: 0.6 """ from skl2onnx.algebra.onnx_operator import OnnxSubEstimator -from .onnx_variable import MultiOnnxVar +from .onnx_variable import MultiOnnxVar, OnnxVar + + +def linear_regression(x, *, model=None): + "See :epkg:`sklearn:linear_model:LinearRegression`." + return OnnxVar(model, x, op=OnnxSubEstimator) def logistic_regression(x, *, model=None): diff --git a/mlprodict/npy/onnx_variable.py b/mlprodict/npy/onnx_variable.py index dc817b26d..8bc704402 100644 --- a/mlprodict/npy/onnx_variable.py +++ b/mlprodict/npy/onnx_variable.py @@ -26,7 +26,7 @@ OnnxSqueeze, OnnxSub, OnnxTopK, OnnxTranspose ) -from skl2onnx.algebra.onnx_operator import OnnxOperatorItem, OnnxOperator +from skl2onnx.algebra.onnx_operator import OnnxOperatorItem from skl2onnx.common.data_types import _guess_numpy_type from ..tools.onnx2py_helper import guess_proto_dtype @@ -486,7 +486,7 @@ def flatten(self, axis=0): return fl -class TupleOnnxAny(OnnxOperator): +class TupleOnnxAny: """ Class used to return multiple @see cl OnnxVar at the same time. @@ -533,16 +533,6 @@ def __getitem__(self, i): return self.unique[i] return self.values[i] - @property - def output_names(self): - "Returns 'output_names' of attribute 'unique'." - if self.values is None: - if hasattr(self.unique, 'to_onnx'): - return self.unique.output_names - raise NotImplementedError( - "Not implemented yet unique=%r values=%r." % ( - self.unique, self.values)) - def get_output_type_inference(self, input_shapes=None): """ Returns the expected output types in a list. @@ -564,6 +554,16 @@ def outputs(self): "Not implemented yet unique=%r values=%r." % ( self.unique, self.values)) + @property + def output_names(self): + "Returns 'output_names' of attribute 'unique'." + if self.values is None: + if hasattr(self.unique, 'to_onnx'): + return self.unique.output_names + raise NotImplementedError( + "Not implemented yet unique=%r values=%r." % ( + self.unique, self.values)) + @output_names.setter def output_names(self, value): """ @@ -615,7 +615,7 @@ def add_to(self, scope, container, operator=None, run_converters=False): raise RuntimeError( "Attributes 'unique' and 'values' cannot be both null.") - def to_onnx(self, *args, **kwargs): + def to_onnx(self, *args, **kwargs): # pylint: disable=W0222 "Converts the underlying class into an ONNX graph." if self.values is None: if hasattr(self.unique, 'to_onnx'): From 905ac408f24e9a7bcf8e1684708bfb3614021272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 25 Mar 2021 23:31:29 +0100 Subject: [PATCH 06/10] fix issue with operators --- _unittests/ut_npy/test_custom_regressor.py | 5 +++-- _unittests/ut_onnxrt/test_cpu_ops.py | 5 ++++- .../ut_onnxrt/test_custom_runtime_ops.py | 2 +- .../ut_onnxrt/test_onnxrt_python_runtime_.py | 10 ++++----- .../onnx_conv/onnx_ops/onnx_tokenizer.py | 4 ++-- .../onnxrt/ops_cpu/op_dequantize_linear.py | 21 +++++++++++-------- mlprodict/onnxrt/ops_cpu/op_pow.py | 2 +- .../onnxrt/ops_cpu/op_quantize_linear.py | 11 ++++++---- 8 files changed, 35 insertions(+), 25 deletions(-) diff --git a/_unittests/ut_npy/test_custom_regressor.py b/_unittests/ut_npy/test_custom_regressor.py index 5151a09ea..343151b52 100644 --- a/_unittests/ut_npy/test_custom_regressor.py +++ b/_unittests/ut_npy/test_custom_regressor.py @@ -52,7 +52,7 @@ def custom_linear_regressor_converter(scope, operator, container): dtype = guess_numpy_type(X.type) m = OnnxAdd( OnnxMatMul(X, op.coef_.astype(dtype), op_version=opv), - op.intercept_, op_version=opv) + numpy.array([op.intercept_]), op_version=opv) Y = OnnxIdentity(m, op_version=opv, output_names=out[:1]) Y.add_to(scope, container) @@ -85,7 +85,8 @@ def fit(self, X, y=None, sample_weights=None): return self def onnx_predict(self, X): - return nxnp.identity(X @ self.coef_.astype(X.dtype) + self.intercept_.astype(X.dtype)) + return (nxnp.identity(X @ self.coef_.astype(X.dtype) + + self.intercept_.astype(X.dtype))) class TestCustomRegressor(ExtTestCase): diff --git a/_unittests/ut_onnxrt/test_cpu_ops.py b/_unittests/ut_onnxrt/test_cpu_ops.py index 895cfa961..b674c3bf5 100644 --- a/_unittests/ut_onnxrt/test_cpu_ops.py +++ b/_unittests/ut_onnxrt/test_cpu_ops.py @@ -25,7 +25,7 @@ def setUp(self): logger = getLogger('skl2onnx') logger.disabled = True - @ignore_warnings(DeprecationWarning) + @ignore_warnings((DeprecationWarning, FutureWarning)) def test_cpu_conv(self): x = numpy.array([[[[0., 1., 2., 3., 4.], # (1, 1, 5, 5) input tensor @@ -56,6 +56,7 @@ def test_cpu_conv(self): [72., 111., 117., 123., 84.]]]]).astype(numpy.float32) self.assertEqualArray(exp, got) + @ignore_warnings((DeprecationWarning, FutureWarning)) def test_cpu_conv_init(self): x = numpy.random.rand(1, 96, 56, 56).astype(numpy.float32) W = numpy.random.rand(24, 96, 1, 1).astype(numpy.float32) @@ -86,6 +87,7 @@ def test_cpu_conv_init(self): ii, diff[ii], gotrt['Y'].ravel()[ii], got['Y'].ravel()[ii])) self.assertEqualArray(gotrt['Y'], got['Y'], decimal=5) + @ignore_warnings((DeprecationWarning, FutureWarning)) def test_cpu_conv_group(self): x = numpy.random.rand(1, 3, 3, 4).astype(numpy.float32) W = numpy.random.rand(9, 1, 3, 3).astype(numpy.float32) @@ -138,6 +140,7 @@ def test_cpu_conv_group(self): ii, diff[ii], gotrt['Y'].ravel()[ii], got['Y'].ravel()[ii])) self.assertEqualArray(gotrt['Y'], got['Y'], decimal=5) + @ignore_warnings((DeprecationWarning, FutureWarning)) def test_slice_bug(self): for opset in [9, 12, TARGET_OPSET]: diff --git a/_unittests/ut_onnxrt/test_custom_runtime_ops.py b/_unittests/ut_onnxrt/test_custom_runtime_ops.py index c3ad3653c..a49174a65 100644 --- a/_unittests/ut_onnxrt/test_custom_runtime_ops.py +++ b/_unittests/ut_onnxrt/test_custom_runtime_ops.py @@ -66,7 +66,7 @@ class OnnxEig(OnnxOperator): """ since_version = 1 - expected_inputs = ['X'] + expected_inputs = [('X', 'T')] expected_outputs = ['EigenValues', 'EigenVectors'] input_range = [1, 1] output_range = [1, 2] diff --git a/_unittests/ut_onnxrt/test_onnxrt_python_runtime_.py b/_unittests/ut_onnxrt/test_onnxrt_python_runtime_.py index b993d0a7a..dd300a26a 100644 --- a/_unittests/ut_onnxrt/test_onnxrt_python_runtime_.py +++ b/_unittests/ut_onnxrt/test_onnxrt_python_runtime_.py @@ -1328,8 +1328,8 @@ def test_onnxt_runtime_dequantize_linear(self): self.assertEqualArray(exp, got['Y']) X = numpy.array([0, 3, 128, 255]).astype(numpy.uint8) - x_scale = numpy.float32(2) - x_zero_point = numpy.uint8(128) + x_scale = numpy.array([2], dtype=numpy.float32) + x_zero_point = numpy.array([128], dtype=numpy.uint8) exp = numpy.array([-256, -250, 0, 254], dtype=numpy.float32) onx = OnnxDequantizeLinear( 'X', x_scale, x_zero_point, output_names=['Y'], @@ -2049,8 +2049,8 @@ def test_onnxt_runtime_quantize_linear(self): self.assertEqualArray(exp, got['Y']) X = numpy.array([0, 2, 4, 1000, -254, -1000]).astype(numpy.float32) - y_scale = numpy.float32(2) - y_zero_point = numpy.uint8(128) + y_scale = numpy.array([2], dtype=numpy.float32) + y_zero_point = numpy.array([128], dtype=numpy.uint8) exp = numpy.array([128, 129, 130, 255, 1, 0]).astype(numpy.uint8) onx = OnnxQuantizeLinear( 'X', y_scale, y_zero_point, output_names=['Y'], @@ -3161,5 +3161,5 @@ def test_make_constant(self): if __name__ == "__main__": - # TestOnnxrtPythonRuntime().test_onnxt_runtime_conv_transpose_B() + # TestOnnxrtPythonRuntime().test_onnxt_runtime_pad() unittest.main() diff --git a/mlprodict/onnx_conv/onnx_ops/onnx_tokenizer.py b/mlprodict/onnx_conv/onnx_ops/onnx_tokenizer.py index 5951983f9..4a9f3a21e 100644 --- a/mlprodict/onnx_conv/onnx_ops/onnx_tokenizer.py +++ b/mlprodict/onnx_conv/onnx_ops/onnx_tokenizer.py @@ -12,8 +12,8 @@ class OnnxTokenizer_1(OnnxOperator): """ since_version = 1 - expected_inputs = ['text'] - expected_outputs = ['tokens'] + expected_inputs = [('text', 'T')] + expected_outputs = [('tokens', 'T')] input_range = [1, 1] output_range = [1, 1] is_deprecated = False diff --git a/mlprodict/onnxrt/ops_cpu/op_dequantize_linear.py b/mlprodict/onnxrt/ops_cpu/op_dequantize_linear.py index 91f5c3097..d4051f240 100644 --- a/mlprodict/onnxrt/ops_cpu/op_dequantize_linear.py +++ b/mlprodict/onnxrt/ops_cpu/op_dequantize_linear.py @@ -24,26 +24,29 @@ def _run(self, *args): # pylint: disable=W0221 raise RuntimeError( # pragma: no cover "Input 2 must be a vector or a number.") + x_scale = args[2] + if len(x_scale.shape) > 0 and x_scale.size == 1: + x_scale = x_scale[0] if len(args) > 2: - if args[2].dtype != args[0].dtype: + if x_scale.dtype != args[0].dtype: raise RuntimeError( # pragma no cover "Type mismatch {} != {} in DequantizeLinear.".format( - args[0].dtype, args[2].dtype)) + args[0].dtype, x_scale.dtype)) - if len(args[2].shape) > 0: + if len(x_scale.shape) > 0: new_shape = [1 for s in args[0].shape] - new_shape[self.axis] = len(args[2]) - x = args[0].astype(numpy.float32) - args[2].reshape(new_shape) + new_shape[self.axis] = len(x_scale) + x = args[0].astype(numpy.float32) - x_scale.reshape(new_shape) y = x * args[1].reshape(new_shape) else: - x = args[0].astype(numpy.float32) - args[2] + x = args[0].astype(numpy.float32) - x_scale y = x * args[1] elif len(args[1].shape) > 0: new_shape = [1 for s in args[0].shape] - new_shape[self.axis] = len(args[2]) - y = args[0].astype(numpy.float32) * args[2].reshape(new_shape) + new_shape[self.axis] = len(x_scale) + y = args[0].astype(numpy.float32) * x_scale.reshape(new_shape) else: - y = args[0].astype(numpy.float32) * args[2] + y = args[0].astype(numpy.float32) * x_scale return (y.astype(numpy.float32), ) def _infer_shapes(self, *args): # pylint: disable=W0221 diff --git a/mlprodict/onnxrt/ops_cpu/op_pow.py b/mlprodict/onnxrt/ops_cpu/op_pow.py index b87b73cd7..7819d1a4b 100644 --- a/mlprodict/onnxrt/ops_cpu/op_pow.py +++ b/mlprodict/onnxrt/ops_cpu/op_pow.py @@ -14,7 +14,7 @@ def __init__(self, onnx_node, desc=None, **options): OpRun.__init__(self, onnx_node, desc=desc, **options) def _run(self, a, b): # pylint: disable=W0221 - return (numpy.power(a, b), ) + return (numpy.power(a, b).astype(a.dtype), ) def _infer_shapes(self, x, b): # pylint: disable=W0221 """ diff --git a/mlprodict/onnxrt/ops_cpu/op_quantize_linear.py b/mlprodict/onnxrt/ops_cpu/op_quantize_linear.py index ce62a700c..7c3f9460b 100644 --- a/mlprodict/onnxrt/ops_cpu/op_quantize_linear.py +++ b/mlprodict/onnxrt/ops_cpu/op_quantize_linear.py @@ -23,15 +23,18 @@ def _run(self, *args): # pylint: disable=W0221 if len(args[1].shape) > 1: raise RuntimeError( # pragma: no cover "Input 2 must be a vector or a number.") - if len(args[1].shape) > 0: + y_scale = args[1] + if len(y_scale.shape) > 0 and y_scale.size == 1: + y_scale = y_scale[0] + if len(y_scale.shape) > 0: new_shape = [1 for s in args[0].shape] - new_shape[self.axis] = len(args[1]) + new_shape[self.axis] = len(y_scale) x = args[0] / args[1].reshape(new_shape) else: - x = args[0] / args[1] + x = args[0] / y_scale if len(args) > 2: dtype = args[2].dtype - if len(args[1].shape) > 0: + if len(y_scale.shape) > 0: x += args[2].reshape(new_shape) else: x += args[2] From e02f3530e04a72511653ae862cb9ee49d2a9a301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Fri, 26 Mar 2021 01:36:01 +0100 Subject: [PATCH 07/10] update notebooks and documentation --- _doc/notebooks/numpy_api_onnx_ccl.ipynb | 195 +++++++++++++++++++++--- _doc/notebooks/numpy_api_onnx_ftr.ipynb | 106 ++++++------- mlprodict/npy/onnx_numpy_wrapper.py | 8 +- mlprodict/npy/onnx_sklearn_wrapper.py | 38 +++-- 4 files changed, 261 insertions(+), 86 deletions(-) diff --git a/_doc/notebooks/numpy_api_onnx_ccl.ipynb b/_doc/notebooks/numpy_api_onnx_ccl.ipynb index cc51aab6c..3a662fcac 100644 --- a/_doc/notebooks/numpy_api_onnx_ccl.ipynb +++ b/_doc/notebooks/numpy_api_onnx_ccl.ipynb @@ -182,7 +182,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAoaElEQVR4nO2de3Ad9ZXnvwe97kUPIPYlm8UPwTibgrAUxnIyqZANDIawM6khUOGh2qGSsRhgapyAd5MJCSlndxhSk0oKMENtBLWinJmJlCxxNpmaZaPgkN0ZapeVZUxeMpnyghWUSnKvDXZsI9mSdfaPvm3dR/e93X378evb309Vl6RWP06/zvn9zjm/8xNVBSGEkOxxTtICEEIISQYaAEIIySg0AIQQklFoAAghJKPQABBCSEbpTFoAP6xevVoHBweTFoMQQlLFvn37DqtqoXZ9qgzA4OAgpqenkxaDEEJShYjMOq2nC4gQQjIKDQAhhGQUGgBCCMkoqYoBEEJIKywuLmJubg4LCwtJixIJuVwOa9asQVdXl6ftaQAIIZlhbm4O/f39GBwchIgkLU6oqCqOHDmCubk5XHzxxZ72oQuIEGIupRKwd6/1MwQWFhawatWqtlP+ACAiWLVqla/eDQ0AIcRMJiaA9euB66+3fk5MhHLYdlT+Nn6vjQaAEGIepRIwMgLMzwPHjlk/R0ZC6wkQCxoAQoh5HDoEdHdXr+vqstZnkNdeew3vfe97sWHDBtx+++04ffp0KMelASCEmMfgIFCr5BYXrfUZ5DOf+Qy2b9+OgwcP4oILLsDY2Fgox6UBIISYR6EAjI0B+TwwMGD9HBuz1sdMmHHoHTt24LHHHjv794MPPoidO3c23EdV8fzzz+OjH/0oAOBjH/sYvvOd77QuDJgGSggxleFhYMsWy+0zOJiI8p+YsEIP3d1Wh2RszBIrKFu3bsUtt9yC+++/H8vLy/jGN76B559/HldeeaXj9uPj47jwwgtx/vnno7PTUtdr1qzBL3/5y+BCVEADQAgxl0IhEcUPVMeh5+etdSMjlk0KKtLg4CBWrVqF/fv34ze/+Q02btyI9evX4+WXX3bd5/Dhw8FO5gEaAEIIccCOQ9vKH1iJQ7dik+666y7s2rULv/71r7F161YcP34cH/jABxy3HR8fx6WXXoqjR49iaWkJnZ2dmJubw0UXXRRcgApoAAghxIGo4tA333wzduzYgcXFRYyPj6Ojo6NhDwAArr32WnzrW9/CHXfcga997Wu46aabWhOiDIPAhBDiQFRx6O7ublx77bW47bbb0NHR4WmfL33pS3jkkUewYcMGHDlyBCMjI60JUYY9AEIIcSGKOPTy8jJefPFFPPPMM573ueSSSzA1NdX6yWtgD4AQQhpQKACbN4ej/GdmZrBhwwZcd911eOc739n6AVskEz2AUinRTDJCCAEAXHbZZXj11VeTFuMsbd8DiKieFCGEpJ62NgCsJ0UIIe60tQFgPSlCCHGnrQ0A60kRQog7bW0ADKonRQghgXniiSewYcMGiEiopSHaPgvIgHpShBDSEu9///vx4Q9/GNdcc02ox217AwAkWk+KEJJ2Qswj37FjB972trfh/vvvB2CVg77wwgtx3333Ndxv48aNLZ3XjUwYAEIICUTI9aCDlIO+7LLLAp+vGTQAhBDiRAT1oIOUg46SxAyAiKwF8DcA3g5AATylqo2nxiGEkLiIqB6033LQ7doDWALwH1T1JRHpB7BPRJ5T1ZkEZSLEH6wz0r5ElEcepBx0VCSWBqqqv1LVl8q/HwdwAEA4sxwQEgesM9LeRJRHHqQc9OOPP441a9Zgbm4OV1xxBe66666WZLARVQ3lQC0JITII4B8BXK6qv635390A7gaAdevWbZqdnY1fQEJqKZUspV/pHsjngdlZ9gQM5sCBA7j00kv97RRyL295eRlXXXUVnnnmmUgqgjpdo4jsU9Wh2m0THwgmIn0AdgO4v1b5A4CqPqWqQ6o6VOCHRUyBdUayQ4j1oFkOugIR6YKl/L+uqt9OUhZCfME6IyQALAddRkQEwBiAA6r6SFJyEBII1hlJLSa4vaPC77Ul2QN4P4A7AfxERF4ur/ucqj6bnEiE+IB1RlJHLpfDkSNHsGrVKlht0PZBVXHkyBHkcjnP+yRmAFT1BQDt9QRI9mCdkVRhZ9KU2nRSkFwuhzVr1njeniOBCSGZoaurCxdffHHSYhhD4llAhJCEKJWAvXs5RV6GoQEwDH6TJBY4iI2ABsAo+E2SWOBk2aQMDYAh8JskscFBbKQMDYAh8JskscFBbKQMDYAh8JtsQ0wN6HAQGylDA2AI/CbbDNMDOsPDVuG6PXusny3MckXSixHVQL0yNDSk09PTSYsRKSwv3wawUigxDLdqoBwIZhgcWNoGRDSTFCFhQxcQIWHDgA5JCTQAhIQNAzokJdAFREgUsFIoSQE0AIREBQM6xHDoAmoTTE05J4SYCw1AG2B6yjkhxExoAFIOawgRQoJCA5ByWEOIEBIUGoCUw5RzQkhQaABSDlPOCSFBYRpoG8CU83hhvSbSLrAH0CYUCsDmzVRIURN7xhXze0mE0AAQ4pHYM66Y30sihgaAEI/EmnHF/F4SAzQAhHgk1owr5veSGKABIMQjsWZcMb+XxAANACE+iG0mReb3khhgGighPomtyCfze0nE0AAQEid+BxGwpDSJELqASKxkOq097LTOTN9MEgY0ACQ2Mp3WHnZaZ6ZvJgkLGgASC5GmtaehJRxmWifHCJCQoAEggfCrcyNLa09LSzjMtE63m7l/v/mGkBgFDQDxTRCdG0lae5pawmGmdTrdzIUF4KabzDeExChoAIgvgurcSNLa0zZaNqxBBE43U9UyAqYbQmIUTAMlvrB17vz8yjpb5zZT5qGntadxtGxYaZ2VN/PNN4HbbrOUv43Xh0IyTaI9ABF5WkSKIvLTJOUg3mlV54Zatjrro2Xtm7lxY/oMITGCpF1AuwDcmLAMxAfG6dzYajMYjHEPhaQFUdVkBRAZBPAPqnp5s22HhoZ0eno6eqFIUzI3K1YaLjgNMpJEEJF9qjpUuz7pHkBTRORuEZkWkekSg1rGENSVY1LKvmdZ0pJqymnhiE+MNwCq+pSqDqnqUIEvdjopa9pvP1kyRo961ukxpJqaZBRJtjDeAJCUU9a0y9ddjxvvXY8/nJ9IPFPRl06PONU0LZ0L0p7QAJDoqNC05xw/hnMxj6cxgtWwNG1SKftuOn1uv0NTPMJU0zSNYyPtSdJpoBMA/g+Ad4nInIiMJCkPCRkHTbuILgzikPV7DJmKpRKw//slvPn9FcXupNM/Mj+BKz/i0BSPMMMmbePYSPuRqAFQ1WFVfYeqdqnqGlUdS1IeEjIOmrYLizjSNxhLpuLEBPCpiybwrg+txzkfuh5LayzFXqvT1+ZKGJMRiFtTPKJU0zSOYyPtBV1AJDoqNO1y/wDO9OTxxpfH8M3nC5Gn7JdKwGe2lvDVxRGci3mch2PoPD0PLSv2Sp3+o+8eQme+SVM8ggwbpu/HACPsDaEBINEyPIyv/cUsPrCwBxs6Z/Gvdgzj4MHolJz9ve/fD/xOxyGcRrViP3POimK3dfoFGwcTa4pzHFuEMMLelMQHgvmBA8HSx5NPAvfeW70un7eUXdhG4Mkngfvus/zqS0vA+YslHFxaj3OxUrhI83mI08knJiy3T1eXpfzHxqiN00ypZCn9yqJVUb14KSC1A8FIeimVLIVcS2dn+IFO29CcOgUcP25990fOKeCezjG8hTyOYQBL3XmIm4/FhKY43RXhwQi7J1gNlESG/Q2eOlW9/vTpcL0rboampwe481vD+Dm2YBCH0LlxsHHrz2+lzjBLL9g9kO5u6wb57YGwDEQ1jLB7gj0AEhmDg5YrppadO8PVUU6NPcD6/jduBDbeUMAFN4RcIiFM/3KrAwLo666HEXZP0ACQyKj8Bvv6rBb56Chwzz3hnicuQ2N7aA4fCHkEVyvuCo4mc8cEt57h0AVEIiX0SWAcsA3NyAjQ0WH19Hfu9G9obC/KxX0lrD5RLXClh+ZfLxzCD87pRndFcLmlCVhacVe0MkNPFghrAp42hT0AEjlxFKm0G3vPPw+8/rp/5W97UUY/OIFzL1uP0x9ccafUNrJfOTWIpfkQ/cutuCvo6yYtQANA2oZWSlSPjAC98yX89bw1cKx73nKn6MgI5vaXqjw0h1HAn+XGcKYnRP/y8DCwbx/w+OPWT6/uijT7upn1lDg0ACTz2F6UQdQPHPvtfBde++Ghukb2N2UYb+4P0b88MQFs2mSlM23a5C+Qm0ZfNwPXRsCBYCRWQs1WDOlg9pih3vkSZlE9cOwt5HF793fw0Yc34k93FNzHiQWUpVSyqpBe+ZH1Vi0im3YetMRBWrHDgWAkESp7+aE2+p58Eli7FrjuOseD+fEu2F6Uk/kC/rR7ZeDYKXSjA0v4u9O34Y8+vx6/fnTCuZEd8MLs3f79LYfw2/kMDVriIC1zUNXULJs2bVKSHsbHVfN51fPOs352dakCK0s+r1osVu9TLKpOTdWvr2J0tPpANQerPe/4eM3+LicpFlUnJ1XX5oq6BZN6Ern6c0xOVu9XLFrrm12Ygwj2bqtR1JNwOMbMjCWn/bPJMVNDwHtGggNgWh10auJK3c9CA5AenL7x2mVgwNJrNk0Vt33gnp66g53p61edmmquWzycZNs21SFM6Zs4r17o3t7q/aamrGM1ujAHane7HeN6Enld7B2wjr9tm/XTvhj7d8ebkkLs5zAw0F7XZSg0ACRWnPRig0a790bh1JRqf3/dwebRo7tHi431sYeT2Js4tsqd9vPbmi33Pkozxbrd1uaK+sZkucXvZj3DaCl76mbFgClyZAA3A8AYAIkEp/T07m4gl3POVvTsFnYY9qsAPoGd+KPtBfT1NUiL93ASe5PDKGArrHjACfSiLlWicrCV1zTMiljB6k3r8dzIRNVuX3q6XLLixAnn2hauN8UHJmXfxDFAhDTGySqYurAHkC6cevlujT5fDenxcV3qyesx9Ok8evQujFa19MfHVbu7V47T1VX2MPjoAdj/Xo2i/kH3pC7nHOIBtbGARq1Zl3OXZor1uzXyn7XSA6DvPbOALiCSBH56+X7cwqWZol7dM6WrUTyry3I5K0Y7M2P97qjnPJzEcZPRUSv20N8fzGftN1ZgC2FfiN8YgNONDxivIOmHBoCkgqAGo6vLavWfd56lp2sbur29lnHwepKqTewT9fdbBx8d9S94kNa3fczaLKBm53ILdLMHkFloAEhbYqduNss4snWd72QTL0rTU/qShpP50uxczeRl9k0mcTMAHAlMzMbDCNu9e62Y5rFjK+vyeeDMmfqAsO8Bp04HHxiwyi5s3ux/VGsro5e9nKuZvK3KQFJJoJHAIjIgIr/jsP6KMIUjxBGPGStOGUcA8Ld/C/T2Vq/znUTTrNqm31GtrWS+eDmXl+qgzL4hZVwNgIjcBuAVALtF5Gcisrni37uiFoxkHB8TnbhlYl57LbC8XL3t6dPAm282LhFRVUaiWZqni8I93DcYXqFLW6CGOa5l0lwdlMSPk1+o7BZ6GcA7yr+/B5YxuLn89363/aJcGAPIEA4ZK2f6B/Rnu6aajbFydM/XBoprY6P2fq4u9tqDO+1U9qu/sG3cU0jAE7UC2SOEm/nwOciKVAC/QWAAP6n5+x0A9gH4JICX3PaLcqEBiBDTFIZDMPMk8npJf9G3UnULFOfzVkKP33pFjlaiwQjfwIk2bgHddqsNRCLHzQA0igEcr/T/q+qvAFwD4CYA7w6zF0ISxqTRoTYVrozl/gG8hTy2YgyvHi/4nva2UAAuuKDefd7RYZXfr/QyLS5Wb1PnzndzTQHA5s147UQhvEKXbj7/Eyfi8eFHMWELJ4ExikYG4F4AUrlCVY8DuBHAw1EKRWIkhknFA3/zw8M4vG8Wz27fg3f3zuKbWKnB7FepusVG3SouVG5TNbtik0BsXx+wsNDkGDW43p+Ip3ts+FyiaBSY2NDIOk7dAqvHgFcB/DmAjop1bwfwd3DpTkS90AUUARGPDvWaIt9oX4fab65ulUaerNoUeNv9U1n24X2dU7qmp+juYm+QZ28f308Bz6b3xyVvv1WPXcPzRjFgjIPQEsVNZzcyABcAGAXwEwC/B+A+ALMA/gzAOW77RbnQAERAhB9mK4d2K4fT1+euVL0Ym1rFae/zxzmrHPOp/Hm6nM/rwYfGrW2aRZbLJ3KSt6fHcte3fH/sIEZ5LoJWjKqn80bRKGAZikTxbQDObmAp/mUAcwDWNNs+yoUGICIiGh3ayjfvtG9/v+quXe4t/6DGpjRT1KUelwhxLmfVkcjlqu9LjWEIcq2e96nQ+Mv5vN7ZNd6SvW56XvYA2o4gPYDzATwJKx30BgCP2b0Bt32iXmgAIiSCLKCwewCN9m2pgelmbTo6qtd1ddUJUFmuJ0ipH6d9SjMVz8IlG6qyCJ7fhrSnexu0UeDHB8cyFLERxAC8CuBTADor1l0J4H8DmHDbL8qFBiBGQjIIrXzzfvZtqYFZLNaXD62sJ125nK0oFzxFv9E17r51XJcrD/rQQ3XG6SgGdAhTvgyNBy+Wtx29XIwfHxyJhSAGwNXdA+BP3P4X5UIDEBOtOplr8PPNNxpv5VVs38bGaQKBBx5oaABaTdGvvK5i0dLza3pc5gauMU6L3Xldm2sQqHa4J06PMlRd3MgCh3UiGo/ABI4BmLTQAMRAgr7aMOyOrSOq3CjNdnDT5LW9gO7ulnz+tddpV5f+8petvx3nIR4YsKyDQ9C52eXF+ijdbogte6uNiZAbJVnDSAMAa0zBzwEcBPBAs+1pAGIgoWyNUJWVH2XR6Hrt49ROBN+CvG7ZTbmcyzzELbSiay9tNYp6Te+UvjTZ+BiBGtpOF5bL6XIuhIfKAHLLGGcAAHQA+H8ALgHQDeBHAC5rtA8NQED8+mAS+NhCszt+5W+2fcW9c0sh9eNycpnT/uxyO6x01KMYsGIBDQ7a7LFWXtodZ497XsPjttTQrrkhL9/6kB516tH4fahMIW0ZEw3A+wBMVvz9WQCfbbQPDUAAgnzRCWRrhGZ3fCqLYlH14EPlwKuHaSIbFZLzep09PfWKP5ez1g8MqK7NFXX3A1ZL3e24fuagWZtr0LOoka3lZ1C+IaWZouN5l9kDSAQTDcBHAfyXir/vBPCEw3Z3A5gGML1u3brIblBb0moeZswBt1Dsjo9gZKUSXZsr6vcecr7esPXP6Gi9AbDDDi9NWsp/ba4YeNKvWt6YnNLF3vOaGsUwG9r2sSp7NCdhDbCzr8HX68UU0pZIrQGoXNgD8EkKu86h2B0nZVHTZD42Ot48B79MFLfRnmO+amTzuNUTOYZ+nUeP3oVRR+XuJE9fn/sgOa8WI0xDV3ms1SjqECyjViy24GZiFlBgTDQAdAFFTZa7zrV5ljX3YanHKi1deWv+ODdujQiu0UxR3cYqfeZwkmVA78JonbFxCyT39zdQqB5b0GE2tJ2OleVXMklMNACd5cFmF1cEgd/daB8agACw6+w6uczVPSuDqRpm4GgMt3FqSk/l6qPD8+g523KuxJanr6/eELgqVI8t6DAb2rXHMrpT2sY9DOMMgCUTfh/AP5ezgR5stn3bG4CoXsBWj5vGD6NJD0Dzed09Wjyr1K/umdJT+fMaaqYob2NppqjzqI8OH0O/FZuoZGZGddcuPfzCjO7aVZ9V5Euheh1QENLzN7YH0ObjDIw0AH6XtjYApr6ApsrVCCeZm5RVLs1Eq5ma3capKdVP5kZ1ucYALPXUyLBtW9X/T27d5jXmXXW9xaIHobwI3sK9MKZTaqxVCg8aAJMx9QX0IpdpvQMfWUB1RKSZvN7GfF71LozqPHr0OM7Vt2AFq88yM1N9kPIy8YUZ7empjgE0soF21tNit0ehIngvjXptjPZLhYObAWg0IxiJiyazTCVGM7n8zvAUx3SALjK/uf8Q9h4qoDTYYCrF4WFgdhbYs8f6OTzsvF0Zr5fj5fHaM2Ce7hrAMgSAoKtLMTBQsdPUVN2xFcAPvjiF7m5r8rBHHwW2bHGe5G3r1pV1b184hJOnmwgV4XtZKMQzq6UnQpp5LZWzXTpZBVMX9gAMkssls6Y006R1HbUryUEuu3hamKf2fDnFor4xaaVANn28xaI1UMptQ4cewDKg78JM1eaTk/UN2t5ea2kU9F7O56sHn5n6XkZBi70/0z2loAvIcIxzjJZxk8uh23wUVmZNnehxK5IKmcOYQKUWz5dToRUWuy05Gj5eL66IihjAMqBf7dpWt/nkpLN8tRWv7+xaGQFty1enwGpiDrptW/AbZzoB/VJpsJM0AGnAKMdoBW7RxJq33p6opO7lT8LHWpb5pcliMrMbOuX117awnWT2okkqsoDy+ZWBVpX3vtFYuCoj1KCHEnVgvF1IQwiBBqCdScpwjFsDp+xh/rdj3LMyjEuRRHFqT8cMqhV89gSf27pS5O0k8vrCtuqKpQ2zgJqI+rNdKdBsBsAeAA1AciTsfCzNFPXqnqmqKQobukMScHFFceqmx2ygFZraa48GffeotyJvzQ7vJqrJPQDTOsumenBtaADaEUOaHp5f/gS/2ihO3fSYDjcmLHtdLFqD12onkDnT37iF7nZ+12dooGYzNeBqmlGqxM0AiPW/dDA0NKTT09NJi2EOe/daKZjHjq2sGxiw0hg3b45VlFLJyg4cHDQktc8ASiVgbn8JgziECzYOooQC1q+3UjFt8nkr49TvPdu7F7jjuhJ+cnw9zsXKAc/05NHxuvMBSyU0PL/rMzTo4Ta7BuKMiOxT1aHa9RwHkGZCyl8OA6PyukMkaG63PUTi2tsKuOgjmzGxpxBqWv3gIPCrpQK2YgxvIY9jGMBbyOPkzjHXh+B0/o4O4NlnretzfYYGPVxTh8ykFRqANGOPHsrnrZZ/Pm/9bcCH2jIGjKrxO87NplRyHojV1+dur/1erv3o/z4/jMv7ZvEH3Xsw+sAsTt3iPnjNqb1w4gTwiU/4u74kMajN0x44+YVMXRgDcMFk52MQ/Dp5y2mROjMTmgithFe8TDPcYKoCXz7tYtHfvOvNqojOzJj/KhkYljAeMAhMYiGMkpl+NG9EA5Vaye1udgkeCpV6vn0zM/VTTDbbf2ZGdceO6pHBwMq0lKYFV51otzZP1LgZALqASGtU+i6C+kwq8ePkPXAAeOKJ6nVPPGGtb5FWXA3NPHOVLvVWfNoTE8DGjcCpU973n5gAPnRVCf/rK3uRP1ntb1pYsI5V6bYyta6NQWGJVEMDQIJTqfDXrQM+/vF6x7dfDeJH8zoUR2u43gethle81pULamjsOEOt8m+0f6kEfO9jE3hlYT2+89b1mMV63I4J9PUBPT3WNVbS2Qn84BslvPl9b8EJA8I2xC9O3QJTF7qADMLJd1G7BB016tXJ61IeOexYQNSuhiA+bScXFWC5cNz2f2myfuDYSeR14vGizszUP847sDLKeLG7sWBh5ObTrRMdYAyAhIqbBvLqjG72tXvVBm1SrMyv8nOyvz09jW3fG5NTerRm4NhRDOgbk5aRtpV4T497tVAnAcMYj2jq4K52gQaAhIvTV9/V5a0pG/bXHkEWUBrw0nOoMizF+klgFrurNfXMjGp3t+oQ6kcZL/Y69+haLYaW6ID2jHQ7aABIY4J8CE4ayEvL3oDyFe1CbUZR5a13tLPjVgnoxV6rVHat1ZiasmYWc+sBlGaKXgrD+nqkiVXTzFC3gwaAuNNqIrofw5GG2rkpw2kswOhoA6Xc4JlVKvPbz8YABnSxy6o06vaatJKbn0ibIGMNERoA4kzcH0LGPryoGR+vn+jFjgf09wezs7Yy7+9Xvai7qLsfmNLSTLHpY2vFmxL74K6MNUTcDEBnkhlIxADsRPTK6lp2InkUSdZ2fuXIiJVnaE9ky4Ru39ipoAsL9f/r6go+jmF42JpX2Kr/VkChUMBf/mX1K2Kfo/I1KRSCP8bqc8bwOrCmBACOAyBJfAjDw5bSP33aMj7bt5tRiCZliexOg8hszpwBdu4MPo6hcqBVqQQ8/HD9NouLVn2jsG6Z18FdoTymdq6j5QenboGpC11AERF3/9tEN1AKA4JuQzEqxQ8jyWVysr5sBKB6663xhY5sQn9MzAJKXrF7XTJjAJJ4KeM8p2n+VxMNkkcqbXcuZwWDwxTbPr5TjCHoLQuqxFP8mBLHzQDQBWQaYdTTCUKcxVVM87+muMh8ZcmJX/wC+Pznw3uElWWtK8nlrPMEuWVupbK9uHNS/JhaJirvJA2ASbTydaQJ0/yvphkkn0Rlu50Ua3c38N3vAvfcE+yW+VbiFZov5Y8pMFG2CWkATCJLTRyv1dLiwDSDFBF+W5F9ffWt/9OngR//OPgt86XEazRfYc9E3TkffdT6PGJrI8WcKBB5m9DJL2Tq0vYxADo5k6WNA4JB/O5TU/VzDdj+f6e5DfzK0jDnoMG3YJ/THuwWW9w+gUSBsMJlYBA4JXC6IxIyQdsVxaKzAejvbz1e39RwNNF8sbeVEmqchXVaNwNAF1DU+O0ymuQaIf4xcCxBM8+im8iFgjWWoJalpdb97k3jFk18RbF7SxNyz0bunXSyCqYuqesBpDC3nLSAoc+7USvSi8ijoyulJWK9rAa94az0ACpP34p3EnQBxQz9+dnC8Oddq0t3jxb1jckpXZsrehI5sfBIgxPH7i1NsXvWzQCwFlBUxF1jx41SKcYCKxnGlOftQmWtnXe9NIGB7SNYOqcbryycxlaM4ZuwXI1uIrdS56clGpw49vpBsZ8wemgAosKEpOWJCStnrLvbkmVsjDGFqDDheTehUAAKKAEftPIKOzGPTgBPYwQ/wBYcRsE0kZsSu2FKzBJGA4PAUZF0bnlWBpWZQtLP2ysOwcxFdOHy3kPGikyiI5EegIjcCuA/ArgUwHtUdToJOSInyS6j4S6JtiQNLgKHnspAfhGPfHsQazaaKTKJjqRcQD8FcAuAJxM6f7g08rMn1WVMgUuiLTHdRVA5H0NXF7C4CBkbw8YbDJaZREYiLiBVPaCqP0/i3KGTVPG2ZqTFJUHip13Gmhg45iJtiJUhlNDJRf4ngE81cgGJyN0A7gaAdevWbZqdnY1JOg+USpbSr3Sz5PPWR2WKomUWEGlHmODgCxHZp6pDtesj6wGIyB4R+anDcpOf46jqU6o6pKpDBdMUWBqKt8VZ5pmQOGCCQ2hEFgNQ1S1RHdsY6GcnJH6Y4BAaTANtBfrZCVkhLp88G16hkYgBEJGbRWQOwPsA/HcRmUxCjlBol4AaIa0QZzIEG16hkWgQ2C9DQ0M6Pd2eQwYISS1JJUMwwcEzbkFgloIghLRGUj5508dcpADGANoZLz5Z5lKTVvHpk+crZw40AO2KF5+sqYPYSLrw4ZPnK2cWjAG0I158smkYxEbSRROfPF+55Ih9IBhJEC8D1NIwiI2kiyaDDk175eiKogFoT7z4ZJlLTWLGpFeOrigLGoB2xItPlrnUJGZMeeUaVZLIWq+AMYB2xkueNHOpScwk/crt3Wu1/I8dW1k3MAB8+tPAF7/YnvXl3GIANACEkEzhFoxWBRYWqte1S4CaQeCoyFqfkZCUU+mK6u21fn7uc0BPT/V2WciJoAFoBUaSCEkttvND1TIKpgSo44QGICisSU5I6MTRobY/3YUF4ORJ6+f27cCjjyYfoI4bGoCgmJbUTEjKiatD7fbpXnVV9gr7shhcUExKaiYk5VR2qO3g7MgIsGVLOK3wysyjRp9u1urLsQcQFFOSmglpA6LsUNf2LPbs4adrwzTQVkk6qZmQNiCqOkGNjgtk59NlGmhUcNJ1Qlomqg51o54FP13GAAghhjA8bPn8w2yVM1TXGPYACCHGEHarnKG6xrAHQAhpa6LoWbQLNACEkLYna+mdXqELiBBCMgoNACGEZBQaAEIIiRhTiwbTABBCSISYXDSYBoAQQiLC9KLBNACEEBIRphcNpgEghJCIMH0kMg1AUEyN6hBCjMH0kcg0AEEwOapDCDGK4WFzJ5phOWi/RFW3lhBCIoLloMPC9KgOIYR4hLWAvGJP/NLXZ3ZUhxBCPMIegBcqff6bNlmJvKZGdQghxCPsATTDabbqsTFg3z7gxAnWlyWEpJZEegAi8mUReUVEfiwi/01Ezk9CDk+4+fxPnOB8coSQVJOUC+g5AJer6hUA/hnAZxOSozmmj+QghJCAJGIAVPX7qrpU/vNFAGuSkMMTpo/kIISQgJgQA9gK4JtJC9EQzilHCGlDIjMAIrIHwL9w+NeDqvrd8jYPAlgC8PUGx7kbwN0AsG7duggk9QjnlCOEtBmRGQBV3dLo/yLycQAfBnCdNhiOrKpPAXgKsEYChykjIYRkmURcQCJyI4A/B/BBVX0rCRkIISTrJJUF9ASAfgDPicjLIjKakByEEJJZEukBqOqGJM5LCCFkBZaCIISQjJKqctAiUgIwm6AIqwEcTvD8YdEO18FrMANegxk0u4b1qlqXxpgqA5A0IjLtVFM7bbTDdfAazIDXYAZBr4EuIEIIySg0AIQQklFoAPzxVNIChEQ7XAevwQx4DWYQ6BoYAyCEkIzCHgAhhGQUGgBCCMkoNAA+EZGHyjOZvSwi3xeRf5m0TH5J1YxsLojIrSLyMxFZFpFUpfCJyI0i8nMROSgiDyQtTxBE5GkRKYrIT5OWJQgislZEfigiM+X36L6kZfKLiOREZEpEflS+hv/k+xiMAfhDRAZU9bfl3z8J4DJVvTdhsXwhIjcAeF5Vl0TkSwCgqp9JWCxfiMilAJYBPAngU6o6nbBInhCRDliz4F0PYA7AXgDDqjqTqGA+EZF/A+AEgL9R1cuTlscvIvIOAO9Q1ZdEpB/APgAfSdNzEBEB0KuqJ0SkC8ALAO5T1Re9HoM9AJ/Yyr9ML4DUWdBUzcjmgqoeUNWfJy1HAN4D4KCqvqqqpwF8A8BNCcvkG1X9RwBvJC1HUFT1V6r6Uvn34wAOALgoWan8oRYnyn92lRdf+ogGIAAi8rCIvA7g3wHYkbQ8LbIVwP9IWogMcRGA1yv+nkPKFE+7ISKDADYC+L8Ji+IbEekQkZcBFAE8p6q+roEGwAER2SMiP3VYbgIAVX1QVdfCmslsW7LSOtPsGsrbNJ2RLUm8XAMhrSAifQB2A7i/pnefClT1jKpeCasX/x4R8eWOM2FOYONoNptZBV8H8CyAL0QoTiDCmpEtSXw8hzTxSwBrK/5eU15HYqbsN98N4Ouq+u2k5WkFVT0qIj8EcCMAz4F59gB8IiLvrPjzJgCvJCVLUCpmZPtDzsgWO3sBvFNELhaRbgB3APj7hGXKHOUA6hiAA6r6SNLyBEFECnYGn4jkYSUW+NJHzALyiYjsBvAuWBkoswDuVdVUteBE5CCAHgBHyqteTGEm080A/hpAAcBRAC+r6ocSFcojIvL7AB4D0AHgaVV9OFmJ/CMiEwCugVWG+DcAvqCqY4kK5QMRuRrAPwH4CaxvGQA+p6rPJieVP0TkCgBfg/UenQPgv6rqX/g6Bg0AIYRkE7qACCEko9AAEEJIRqEBIISQjEIDQAghGYUGgBBCMgoNACE+KFeRfE1E3lb++4Ly34Mi8j0ROSoi/5C0nIR4gQaAEB+o6usAvgrgr8qr/grAU6p6CMCXAdyZkGiE+IYGgBD/PArgd0XkfgBXA/gKAKjqDwAcT1AuQnzBWkCE+ERVF0Xk0wC+B+AGVV1MWiZCgsAeACHB+LcAfgUgdZOhEGJDA0CIT0TkSliFt34XwPby7FKEpA4aAEJ8UK4i+VVY9eN/ASvw+5VkpSIkGDQAhPjjTwD8QlWfK//9nwFcKiIfFJF/AvAMgOtEZE5EUlGdlGQXVgMlhJCMwh4AIYRkFBoAQgjJKDQAhBCSUWgACCEko9AAEEJIRqEBIISQjEIDQAghGeX/A5vQ4dfeS5N/AAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEJCAYAAACdePCvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmH0lEQVR4nO2df5Ad1XXnv0eaN/MmGg040jjLaoRGRJQjxUUQHrCz2LsmAi+hNsG4HGBqC4w1BFNZYiBOYmKqSCWYKlPExoC9GdgMAao8ooJhvVteNiJjnB+kgkcDwl57ZGdZEPa4bL+RjIQECI00Z//o95ie97r79e97u/v7qeqaee/16z59X/c99/y454qqghBCSPVYZVoAQgghZqACIISQikIFQAghFYUKgBBCKgoVACGEVBQqAEIIqSjGFICI1EVkRkS+LSLfE5E/MyULIYRUETE1D0BEBMAaVT0qIjUAzwC4UVWfNSIQIYRUjB5TJ1ZH8xxtvqw1t0BttH79eh0ZGclYMkIIKRfPPffcAVUdan/fmAIAABFZDeA5AFsAfFlVv+Wxz3UArgOA008/HbOzs/kKSQghBUdEXvF632gQWFVPqurZAIYBnCci7/bY5wFVHVXV0aGhDgVGCCEkJlZkAanqIQDfBHCxYVEIIaQymMwCGhKRU5v/9wO4CMD3TclDCCFVw2QM4DQADzfjAKsA/I2qft2gPISQkrO4uIj5+XkcO3bMtCiZUK/XMTw8jFqtFmp/k1lA3wGw3dT5CSHVY35+HmvXrsXIyAicTPTyoKo4ePAg5ufnsXnz5lDfsSIGQCxhYQHYs8f5S0gJOXbsGNatW1e6zh8ARATr1q2LZN1QARCHXbuATZuAiy5y/u7aZVoiQjKhjJ1/i6jXRgVAnBH/+Djw5pvA4cPO3/FxWgKElBwqAALs3w/09q58r1Zz3ieEGOfll1/Ge9/7XmzZsgVXXHEFjh8/nspxqQAIMDICtN9Qi4vO+4QQ43z605/GzTffjBdffBHveMc7MDk5mcpxqQAIMDQETE4C/f3A4KDzd3LSeZ+QipNmbsRtt92GL37xi2+/vvXWW3HPPfcEfkdV8fTTT+OjH/0oAOBjH/sYvva1ryUXBoZrARGLGBsDLrzQcfuMjLDzJwROLsT4uOMhPX7cGReNjcU/3s6dO/GRj3wEN910E5aWlvDoo4/i6aefxtlnn+25/9TUFN75znfi1FNPRU+P010PDw/jxz/+cXwhXFABVJWFhc7OfmiIHT8hTdy5EW++6bw3Pu6Mk+I+JiMjI1i3bh327t2Ln/3sZ9i+fTs2bdqEF154wfc7Bw4ciHeyEFABVJG0hzWElJBWbkSr8weWcyOSjJOuvfZaPPTQQ/jpT3+KnTt34siRI/jABz7gue/U1BS2bt2KQ4cO4cSJE+jp6cH8/Dw2bNgQXwAXVABVI4thDSElJKvciMsuuwy33XYbFhcXMTU1hdWrVwdaAABwwQUX4Ktf/SquvPJKPPzww7j00kuTCdGEQeCqwZRPQkKRVW5Eb28vLrjgAlx++eVYvXp1qO/ceeed+MIXvoAtW7bg4MGDGB8fTyZEE1oAVYMpn4SEJovciKWlJTz77LN47LHHQn/njDPOwMzMTPKTt0ELoGow5ZOQSAwNAeeem84jMjc3hy1btmDHjh0488wzkx8wIbQAqghTPgkxwrZt2/DSSy+ZFuNtqACqClM+Cak8dAERQkhFoQIg0eG6AYSUAioAEg2uG0BIaaACIOHhugGEGOFLX/oStmzZAhFJtTQEFQAJDyeREWKE888/H9PT09i0aVOqx6UCIOHhJDJSRVKMecUpBw0A27dvx0gGzxkVAAkPJ5GRqpFyzGvnzp145JFHAODtctAf/vCHcfbZZ3tuc3NzaVyFL5wHQKLBSWSkKmRQODFOOegsoQIg0eEkMlIFMqoHHbUc9LZt22KfqxtUAKR4eC1mQ0jaZBTzilMOOisYAyDFgvMQSF5kFPOKUw763nvvxfDwMObn53HWWWfh2muvTSRDC1HVVA6UB6Ojozo7O2taDGKKhQWn03eb5P39wCuv0BIgodi3bx+2bt0a7UspW5xLS0s455xz8Nhjj2VSEdTrGkXkOVUdbd+XFgApDpyHQEyQYj1oloMmJC6ch0AKjm3loGkBkOLAeQgkBYrk9o5K1GujBUCKBechkATU63UcPHgQ69atg4iYFidVVBUHDx5EvV4P/R0qgCLB9EcHzkMgMWll0iyUtIBhvV7H8PBw6P2pAIrCrl3OLMTeXscPPjnpjIYJIaGp1WrYvHmzaTGsgTGAIsAyzISQDKACKAJMfySEZAAVQBFg+iMhJAOoAIoA0x8JIRnAIHBRYPojISRljCkAEdkI4BEAvwRAATygqt2XxqkyptIfmX5KSCkx6QI6AeBTqroNwPsA/BcRya7wNYkHq28SUlqMKQBV/YmqPt/8/wiAfQA2mJKHeJBV+mmKa6wSQuJjRRBYREYAbAfwLcOiEDdZpJ/SoiDEGowrABEZAPA4gJtU9TWPz68TkVkRmS3r9O1UyGJUnXb6KSe0EWIVRhWAiNTgdP5fUdUnvPZR1QdUdVRVR4cYgPQmq1F12umnnNBGiFUYWxFMnFJ8DwP4uareFOY7XBHMgzxWyUorC4grehFiBBtXBDsfwFUAfkNEXmhulxiUp5jkMapOa0UkTmgjxCqMzQNQ1WcAlKsgtwmKViaCE9oIsQbjQWCSkCKOqlNcY5UQEh+WgigDHFUTQmJABVAWuEoWISQidAERQkhFoQIghJCKQgVAyg9rDxHiCRUAKTesPUSIL1QApLyw9hAhgVABkPLC2kOEBEIFQMpL0WZJE5IzVACkvBRxljQhOcKJYCR9bFpDmLOkCfGFFgBJlzSzbtJK32TtIUI8oQIg6ZFm1g3TNwnJHCqAImLrxKa0sm6YvklILlABFA2bR8ZpZd0wfZOQXKACSIO8RuS2j4zTyrph+iYhuUAFkJQ8R+RFGBmPjTlr/E5PO3/HxqIfg+mbhOSCsUXh42DdovB5L3JetUXVbUonJaTA2LgofPHJe0Se5sjYRCA56jmZvklIplABJMGErzqJi6XVAd9/f/6BZJuD14RUFCqAJBTJV93qgHfsAK6/Pt9AchbBa1tTYQkpEFQASUkj6BmFOCNpdwd85Ejn51kHktN2ldGaICQVGAQuEnGDwHv2OJ3l4cPen2cdSE4zeF21QDghKcAgcBmIO5L2ilUAwNq1+bit0nSVFSEVlpCCwGqgRSJu0LnVAY+PO53l4iJw993AOefkl2KZVlVOThIjJDWoAIqEV0cediRtQ1nkoaHk503SBoSQFTAGUETymCBl+yQs2+WLQxmviVgBYwBlwm+CVFqpkUXIsinbJLEitDkpHVQAZSGtDsT2gnNlhG3uD+d7ZAoVQBlIswNhlk3+sM29qZpVZEDZUQGUgTQ7EGbZ5A/bvJOqWUWGlB0VQBlIswMpUnmLssA276RKVpFBZcc00DKQdmqkDSmjVYNtvpIqWUUtZeee3d5SdhnfB1QAZSHtDiSNnP1uREl7rEKKZB5tXhSqNN/DoLKjC6hMFCk1MorPs2rBQOKQd6FFUxh0AXIiGMmfKAXdWPyNVIUMrVxOBCP2ECXAV6VgIKk2Bix4owpARB4UkYaIfNekHCRnovg8qxQMzAJOpCIBmLYAHgJwsWEZSN5E8XlWPUUySQfO2AnpgvEYgIiMAPi6qr67276MAZQMZgEFs2uXkwXT2+tYQZOT4QOhjJ0QF34xAKaB5k0VOzI/oqQ9Vi1F0j05qNWJj487qb5hlOWTTwI9bY93TrnlpDiYdgF1RUSuE5FZEZldKLofkyY5CUvc4HfrHvv93+9c/5mxE9KG9QpAVR9Q1VFVHR0q8silarVNioSNgdI4wW/3Pebu/PNa+pMUDusVQGmoajqjjZ2rG1utsjjBb697bGAAuO++lROpbP9NSG6YTgPdBeBfALxLROZFZNykPJlSxXRGWzvXFrZbZVFnwnrdYydPApdcsqw4svxNqFgKh1EFoKpjqnqaqtZUdVhVJ03KkylVS2e0vXMFimGVRZkc1O0ey/I3sV3ZE0+YBZQnVar4aLDCYWjKaJUF3WNZ/SZJMpaIURgDyJsiFWxLQhE617yssrxdI373WNzfpJv8RbCkiCfVUAD0TeZPUVxeWVectMk1Euc3CSN/EZQ98cT4TOAoxJoJnGQ2JUlOlSe+2TobN+xvEkX+1nPmrt3P58waqjkTmL5J81RtBq+btH3uaSnTsL9JFPmrFN8qEYEuIBEZFJFf9nj/rOxEShH6JquFba6+NF0jJlxJUeWvSnyrRPgqABG5HMD3ATwuIt8TkXNdHz+UtWCpQN9kdbDJ194irThIlumbQUqzKHEcEh9V9dwAvADgtOb/58FRBpc1X+/1+16W23ve8x6NzNSUan+/6uCg83dqKvoxSkKjoToz4/wtFY2G89sCy1t/vz0XmrThZ2ZUTzll5fUNDjrvJ6H1bJxySvCzUdobpzoAmFWPPjUoBrBaVX/SVBIzInIBgK+LyEYAxYkc0zcJIKdYuKmArw1zDoKuPWkcJAtLNkp8rMpxnJITFAM44vb/N5XBBwFcCuBXM5YrXSrum8xlUm4WLpiwPn3Trr6s3U9ZuGIYHyMIVgDXAxD3G6p6BM4KXndkKRRJl8yf9Sw0jF+nurAAPPWUs7WOb9JXnVfJi7TnKwwMAMeOrXyP8bHKEeQC+hqACRH5vKqeBAAR+SUAnwfwKwD+PHvxSBpkPkBOywXTcqMMDHi7J157zalzv7i4fI6HH3Y6Q1OuvjzdT2m5Ylr+wFXN8V+9DogwwFtBgiyA9wA4A8ALIvIbInIjgBk41TvPy0M4kg6ZD5C7aZgwrhz3iH/79s7PV68GbrhhufNvnWPnzpWWQN6uviTa1UTaqpfvXxV47jlO3KogvgpAVV9V1esB/BWAaQB/BOB8Vf2yqi7lJSBJh0wrHgRpmDD+8XY3yltvrRxRA04ne+JE53dXrTLrt46rXU2lrXr5A/v6gKNH8zk/sQrfUhAiciqAOwG8F8AfA7gEwA4AN6rq03kJ6KaMi8InTpyxqdRCuyxhSwns2eN0hIcPex+3VgM+9Sngc5/r/Ky3F5ifz+7ao5RNiLLAvakSEbaWpyCZ4lcKIsgF9DyA/wtgVFWfUtWbAFwF4LPNhVxIQhIPAm2b/NTuggkbffZyo7jp6QGuvtr5bjv33ptdxxWlfaO4n5JE5ZO6jaanV1pSvb30/VcZr8kBTatgOOCz3/X7LMst1kQwS0k8d8n2yU+q0WRsTUpas2bl/u5JT1NTqvW6s19vr+rEhB2y53XssBO3opy3Xk/vnuGEMWuBz0SwoBjAfMBn/y11TVQxEqdmFiGPO4p/vBWkeOIJZz83raDq2Bjwwx8C//APjtvnE5/ITvYs2zdO3CCNdFOva+rtTeeabLNGSTi8tIKtmzELIIORTSUsgBZR28+G8h15tG+UdkmjHERW11Ske7GiIKoFQJpkNLJJnJpZpEJdUdMzs16kJQx5tG+UdkljMkdW15SWtRQ2vmFb1dci46UVbN1ytwByGNkkNi5aB5ibq57/NQ+fs01+7bQso7SvKY3nJGx8I2kcpKLAxwIw3qlH2XJXAFlVYYyL34NbxYeiKNecRWdri0Jyk0Q5hVUgdDXFhgogDjbdcH4dXhcZbe0vEtHtd7HloouipNIibruHHWjZNiArEH4KgDGAIIaG8M/jk3gD/TiMQbyBfvzzuAE/e1AGSID/tbSJGUE+Z1suOq8icTYRtxRH2PiG6aqvJYQKIICFBeCiyTFswiu4ENPYhFdw0eRY/s9wUIfn81AcGBixo//JImDn1xG4i8iZ7nSLkKZrC2GD00VKfCgIVAABtJ7hAxjCLM7FAQyZeYaDRj4+D8XLR4fM9z9Zjcb9OoKjR+3pdDlajUbYzC8bMsTKhJdfyNYt7xiATSGArkG2Nv+rcdlN5NHnedFh/N02zGcIiy1xE5IJYBA4HlY9wxEfUqOymwrY5XHRUYK7RehYqxasriB+CsC3GqiNmKoGalPBzai4ZQdyvA7TFS/TuFCv45Stmua+fc76C2+9tfxeka+HeBKnGihpUuQlhVuyT0/nnBxjMmCXxg/mF78oU3B3167Ozh8o7vWQyNACMEEGJkXQIY0OWr3WCEjr2rMyzYIaDCiHBeB1jS2KeD0kEFoAtpBBZky3QxodtLpH4+2Cfvaz8VM0s8z3D2qwsqQiel0j4KwOVsTrIbGgBZAnGQzFwxzSCre134izXgcefDBaOl/WFxS2UW0JDMWRxesa+/qAvXuBrVuzkJIYhBaADWQwFA9zSCsGrX4jzmPH0qlrn6ZJE6bBwsYZsq5cGdcS8rrGv/5rdv4VgxZAnhiyANz7Ghu0BvmcBwedKPW558Y/VhYmTdIG27XLUW69vc6ksMnJdCcupdEONlkyJDNoAdhABkPxKIc0ms3kFrQdW+rae50nboPlUQsoDUuoyCluJDG0AEyQcxZQ3gTKsrAA3H8/cMcdTue1uBh/ZBzX9x3iO4nbc88exy1z+PDye1EtnW5YEdzJCZtu8ALiZwEYn90bZSvTovBZEmfyaVoTVkNPKjUxQzakcO7d6nXV22+PIWZeZSmsmqqeEZypnBiwFEQ1iPOspPV8mag/1Gio7t7tbIHnCSmc126tXSO3S16dcxHKTcTFeFGrcmClAgBwMYAfAHgRwC3d9i+KAjD1PMZ5VtJ8vvIu/zM1pdrbu3yuWi2gjw0pnNduidqlzJ1zHnARmFTwUwDGgsAishrAlwH8JoBtAMZEZJspedLC5HokcWKCXt95803HTR+VPCsgt2Ks7vMtLgIf/7hPnDWEcAsLwKuvdlZGaOHZlt3SPBlkTQbLameKySyg8wC8qKovqepxAI8CuNSgPIkxvQhUnGfF6zuAE6ONKrdfcg6Qfir8/v3AKo+79623fJRXF+GeuH8BmzYBl18OLC0BPT2dh+hoy6a2X9pxEU5u3ITX7i/LkmsWYcUklhLjZRbksQH4KIC/cr2+CsCXPPa7DsAsgNnTTz89IwMpHWywVuO4nW+/vdPdkURut9cjTnwhjNfEz1ff1VXjIdzJtafo6+jXKzC14hi33BLQlh4CvI5+fXyCrp5MsMmVZpMsIYFtMYCwCsC92R4DsCVeFfX+bDScbJe05Y7THn4KY8U1NV88PtHQnp54ymthrqEn+jo78PVoKKC6Zs1yYLl9zZmZGdWf757Rk2tPWfH9QxjU9/fNFKlfIFEpaEaSjQrg1wHsdr3+EwB/EvQd2xWAanGz8rKQO6pF5KcwJiaWn7mra1O62Lv8AP7orint64umZCYmVP9dbUYPYaVwhzCoo5jxzfxxP/sb6w09XutUIJsHGlbGJws4aLUPW0Z4MbBRAfQAeAnAZgC9AL4N4FeDvlMEBaBa3IctltwBX4r6vHgpjLVr9e0Ofj0a+jo6D/j4RCO08pqY8D+W2wJol9frWq6qTenr6NdDGHzbhZRH2mvU36igg1b7sMHHGxPrFIAjEy4B8K8A/h+AW7vtXxQFUBlC9CxRLAuvTravz1ECgOooZvTVtlF76wEM6hhbn83N6Qpr4Qp0duB+7iS/Z//ztzT0/X0zunmgkXnnGjeeUtBBq30UuDGtVABRNyoAi4jwMEQZtbYrjJb7J8gCCDqwu9Ps6+uMdaxHQ0cx0zHyb231eqcF0PrOxnrj7c8ys/iaB1+Ya8Tqewo8aLWTgvp4qQBIumTYs7R3qO5n7qpWDCDEAxiUKeTeajX/z26/ffl4U1NODMKxGk5x5MhpyH+ir18/3j8VubkLPGi1lwL6eKkASLrk3LN4ZQF1O5eXjurvdyyBgQHn7113dVoF7aN/txBLIctJJO4ffNJM3ZZK2OYu6KCVpIifAvCY7kJswPrih60JOuPjzhTZVlXPlIV1t8NyEc2hUOfxm+S2dy9w9Ohy227YAFx9NXDixPI+tZqzUJn7NK/u3Y+1q3rRgzdX7thaKhIpLgHQmqLtqvTZ01/Dryztx3f6hiI199gYcOGFlt9PJJisOgQvrWDrVhULILesjTSGqgmPEfT1uO3gNREtaPQ7MeFYA/39Tm2hW27plGdqykn9DIpBeBlFHVZEWHwsrIW5Rkd7FdAjQaKQQocAuoCKQW6elRRuqqQdT5AIcduhdcw1a5aPGSRnKy006DxuWdyZQ0ttQvsVknPHEbqxQtYQ2sv6FE9qp2Sk1CFQARSEXLI2UripvDqeKM96NxHitEOjsbI6KOC89pOn0dCOSWSAEx9wn2dmZqWs6+Gkfj6/u9P37xVPiOqrX9GZpzjPInes104FIKUOwU8BcElIy8ij+OGre/fjxKr4Swl6Fb275hrg9NO7V0FtFc/cuze4cmmcdti7t/M7x48773vht059+3kGBlYuunUAQ3jmrXNR37jSFzs0BNx6a+fxwjStbyFB+FcTTWNFyMwwXRmxLGTcIVABWEbWxQ937QJ+7dIRHH89/k3l1fEcPw4cOxb8rLdKZe/YAfzWbwFvvOEvQh5FIEdGVgZ+W9xzz8rzHD3auZRxve68384nPuHsux4LGMUerMcCTjm+gC2vBpdDjdOZW10p2WrtVCCyfhC8zAJbtyq4gFpk4ToN68uOchy/rd1K9ftOrRYcoI3qVmrP6a/VwhWfa5WcmJgId71BrpZnblieK3AMvXpida2rGyToHGEC5daleFrvnyoYCTsEMAZA2t2J69HQD67p9GWHob3jae9425/1mZnlkg7ura8vxHKOEeWq150gcL2eXvlpr47W83vdtGNAJ+h1jjBudGvjrNZqp+rhpwDE+awYjI6O6uzsrGkxCsvCguOCcfuz+/uBV16JZ1G6U5Onp1dOCbj7buCcc5bTlhcWgI0bO1fbWrsW+MY33Dn+yckqZdrrejvy/ffscQIhhw97H2Rw0PmyzwW7zwGk+3sZwfoJLdVARJ5T1dGOD7y0gq0bLYDkZDkoa41E3eWb3ecIk3JpE34j60DvRggLwCuX34v27KM4aaW2Yq3VUlJAF1BGFPBOzlLkbq7f1qSrtWvt9goEuV66Zua5tWxv74pAxzM3TIXOjHzmGW8d0tcXPiZi463J7ND8oQLIAt7JHYRJW47TMeXZmXVTYqHim26BY1T0nJrynqMAOPGNbmngad+aHpcT67dgbNgMVABpwzvZkyyaxbczy0grtAesvYLlcVxp3ZRj63Lm5oK9SN3KS6T9G7jbv1ZzjJq4ioXlqc1ABZA2vJN9iRVn8OnM/TqzwxPZWV/uWMWVWE7pbE+Xjap/gjrm9nULghRAT0/w5QbdmmnInESxcNxkBiqAtCnqnezRA6Q+kG409Oe7nRFzqGMG+Cu8OrORNQ2nFn8Gbe/+WeMsQBP2UtvTSbvNq2jfguY3+N2afsH5IHbvdlxOfnLEGfMwOzR/qACyoGh3skdHm3oYI+CAofPmXZ2s18ej6FzQPS3ry61wgpagTEJ7O3gpOT//v3vbvdv/HEErq4XVZa1jBMkQVx/aGqAuK1QAWVGUO9mjJ13q79eNde9F0NM6R+uAvnohhCvNPVs3q5G51yVkeR6/c7a29qJ2URVA67hut08Uj2VYq8Rr5nQeFOWxswU/BcBaQEkZ8i/WZRUetVlOrqrhl1fvX/FeonItPvVfXt27378uWIiCNmNjzuSn++5zJo4dwBB2YhJvoB+HMYiTfenWR/nMZ5wJV8cHh3B9bRInejMsSATvci9/8AfB3+ntBbZv737c1q0ZtW6QX6E8NwMDzmS/vGnVlOpWeJCEwEsr2LpZaQEUhZgWQKSRlo8F8PzuRvi8+QA/VPvhW2WZF+bSGQa6rZR63Zlw9fbkrhyGm+7TzM05wd72Efcv/IIj4+MT0WWK4rEMYwGYCHkVNfRmGtAFRLx6gKBOIVZ8oPmlk2sH9URfvx6emIqeNx/tElIhSMYwoqWpI1rX2JKnt3fZjz8zkywDKoqc7W19ww3mQ15MvosHFQBxCJkFFHek1Wiofv6Whp7fO6NnrG283VGk0nG7JlSlPSD361huv717X5tmIN2r3fv6HIvAd4cQP0xcBTU3p/rQQ8vnT1PRxZ0QSAsgOlQAJBJxRlp+WSNRRtJdD57RrOtGo7OiaU9P984mdIcU8uK7tnuMHyZu02XZ5EmOHaaEN1kJFQCJRNSRVpDPuFu2Sdd+MYdhX6PRmXnT09NZwrr9WkL1xxF6u66XGrEtklhyWTV5GscuSk0pW/BTAMwCIp5EXYgoKGvEL9skdDZHDqtL7d/vvepXt8yZrtk1EZdG7NruEX+YuE2XZZMnPfbCAnDzzU5p8SNHuNpkIry0gq0bLYD8Ceu28bMA/BZliTQKzMkCCJo9GxS7CIxvxFzdvutM6pA/TBktAAaCowO6gEjWtHeEb6dRehD5Ic5h1rXfKRJlAUXt7TJwvEdN/2xdR5ZNnuTYDARHhwqA5JLPnsRi6PoQWyR/JML2dhn2bGGuy0v3ZNnkSY5dtCospvFTAFwSsirs2uWzhqF5kVrLSFogUnaEWRrRaznJLktIpile0Zaf5GqT4fFbErLHhDAkZ9yByNYTPj4OXHih0SdnbMwRoRIP8dBQ9wuMWq8hRVqBWbcCaAVmbf1dwjQpCYZZQFUghyyauKRdSmlhwRlIFzIjJGrqVYoY1D3EIFQAVaAiT3cpioS1Kt9NTzt/c/KJGdQ9xCCMAVSFkjvc0/RhV9m3XORrL7LsWeMXA6AFUBUMjSzzIi0vVymsiAQUpbp5O1X/3eJCC4CUgjQsgCJmwhD+bmGgBUBKTRo+bC8rYvVq4MknCxpUrggW5zhYDxUACaRIWTVJvVxesfKjR4Hf+z26FWymIjkOmWBEAYjI74jI90RkSUQ6zBJiB0X0qybxYbutiIGB5fffeMNxL1xzTTEUYdVgBlN8jMQARGQrgCUA9wP4Q1UN5dhnDCA/quxXXVgAHn0U+OQnOz/bvRv40Ifyl4l0h1lA/lg1E1hV9wGAiJg4PQlBEWeGdiOog2j/7F3vCn8swI6Op+odIGcGR8f6GICIXCcisyIyu0D7OzfK5lcNcmd5fbZ9u6Pw3NRqzvvu/TdsAIaHzbvJiuiuI+bJzAUkItMA/o3HR7eq6v9o7vP3oAvIWsoydyzInQX4fzY9Dezc6WQCnTwJPPigU7uofX83JtxkVXbXkXDk7gJS1QuzOjbJh7IUawtyZwH+n3ld/549nfu7MeEmK6O7juQDq4GSQMrgV+3mzgr6rP36vY7l9928KJu7juSHqTTQy0RkHsCvA/hfIrLbhBykGgSlCUZNIWzfv1ZzRt8m0w+ZBkniwlIQpDJEyQKKcizADjdZ1bOAiD9WpYESYoIgd1ZUV1f7/jZ0uGVw15F8sT4NlBBCSDZQARBCSEWhAiCEkIpCBUAIIRWFCoAQQioKFQAhhFQUKgBCCKkoVACEEFJRqACqRpHWeCSEZAoVQJVg0XhCiAsqgKqwsOAU93/zTeDwYefv+DgtAUIqDBVAVWgVjXfjLopPCKkcVABVgUXjCSFtUAFUBRaNJ4S0wXLQVaIsazwSQlKBCqBqsGg8IaQJXUCEEFJRqAAIIaSiUAEQQkhFoQIghJCKQgVACCEVRVTVtAyhEZEFAK8AWA/ggGFxkkD5zVJ0+YHiXwPlz5dNqtqR/lcoBdBCRGZVddS0HHGh/GYpuvxA8a+B8tsBXUCEEFJRqAAIIaSiFFUBPGBagIRQfrMUXX6g+NdA+S2gkDEAQgghySmqBUAIISQhVACEEFJRCqsAROR2EfmOiLwgIk+JyL81LVMUROQuEfl+8xr+u4icalqmKIjI74jI90RkSUQKkw4nIheLyA9E5EURucW0PFEQkQdFpCEi3zUtSxxEZKOIfFNE5pr3zo2mZYqKiNRFZEZEvt28hj8zLVMSChsDEJFBVX2t+f8nAWxT1esNixUaEfkQgKdV9YSI3AkAqvppw2KFRkS2AlgCcD+AP1TVWcMidUVEVgP4VwAXAZgHsAfAmKrOGRUsJCLy7wEcBfCIqr7btDxREZHTAJymqs+LyFoAzwH4cFHaHwBERACsUdWjIlID8AyAG1X1WcOixaKwFkCr82+yBkChNJmqPqWqJ5ovnwUwbFKeqKjqPlX9gWk5InIegBdV9SVVPQ7gUQCXGpYpNKr6jwB+blqOuKjqT1T1+eb/RwDsA7DBrFTRUIejzZe15laovsdNYRUAAIjIHSLyIwD/GcBtpuVJwE4A/9u0EBVgA4AfuV7Po2AdUFkQkREA2wF8y7AokRGR1SLyAoAGgL9T1cJdQwurFYCITIvIdz22SwFAVW9V1Y0AvgLgBrPSdtJN/uY+twI4AecarCKM/IRERUQGADwO4KY2S74QqOpJVT0bjtV+nogUzh3XwuolIVX1wpC7fgXAkwD+NENxItNNfhG5BsB/ArBDLQzGRGj/ovBjABtdr4eb75GcaPrNHwfwFVV9wrQ8SVDVQyLyTQAXAyhkYN5qCyAIETnT9fJSAN83JUscRORiAH8M4LdV9Q3T8lSEPQDOFJHNItIL4EoA/9OwTJWhGUCdBLBPVb9gWp44iMhQK2NPRPrhJBQUqu9xU+QsoMcBvAtOJsorAK5X1cKM5kTkRQB9AA4233q2YFlMlwG4D8AQgEMAXlDV/2hUqBCIyCUAvghgNYAHVfUOsxKFR0R2AfggnFLEPwPwp6o6aVSoCIjI+wH8E4D/A+e5BYDPqOqT5qSKhoicBeBhOPfPKgB/o6p/blaq+BRWARBCCElGYV1AhBBCkkEFQAghFYUKgBBCKgoVACGEVBQqAEIIqShUAIREoFnR8mUR+cXm63c0X4+IyN+KyCER+bppOQkJAxUAIRFQ1R8B+EsAn2u+9TkAD6jqfgB3AbjKkGiERIYKgJDo3A3gfSJyE4D3A/gLAFDVbwA4YlAuQiJhdS0gQmxEVRdF5I8A/C2AD6nqommZCIkDLQBC4vGbAH4CoLCVIAmhAiAkIiJyNpwiYO8DcHNzpStCCgcVACERaFa0/Es4tex/CCfw+xdmpSIkHlQAhETjdwH8UFX/rvn6vwLYKiL/QUT+CcBjAHaIyLyIWF8dlVQbVgMlhJCKQguAEEIqChUAIYRUFCoAQgipKFQAhBBSUagACCGkolABEEJIRaECIISQivL/AVPx0O7CiD/YAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -239,9 +239,9 @@ { "data": { "text/plain": [ - "array([0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0,\n", - " 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0,\n", - " 0, 0, 0, 0, 0, 1], dtype=int64)" + "array([0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1,\n", + " 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0,\n", + " 0, 0, 0, 1, 0, 0], dtype=int64)" ] }, "execution_count": 6, @@ -322,7 +322,7 @@ { "data": { "text/plain": [ - "(0.6, 0.72)" + "(0.9, 0.86)" ] }, "execution_count": 7, @@ -351,8 +351,8 @@ { "data": { "text/plain": [ - "array([[ 0.03522525, 0.03404299],\n", - " [-0.29776838, 0.07071687]])" + "array([[ 0.08857299, -0.35549907],\n", + " [-0.13914217, 0.5407919 ]])" ] }, "execution_count": 8, @@ -372,8 +372,8 @@ { "data": { "text/plain": [ - "array([[0.10947197],\n", - " [0.99398988]])" + "array([[0.96920866],\n", + " [0.2462409 ]])" ] }, "execution_count": 9, @@ -393,7 +393,7 @@ { "data": { "text/plain": [ - "(array([[-0.90862235, -0.21881959]]), array([[1.14845026, 1.08707429]]))" + "(array([[-0.03875925, 1.61780094]]), array([[0.43777242, 2.72237106]]))" ] }, "execution_count": 10, @@ -429,7 +429,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABE2klEQVR4nO2deXxU5bn4v+9s2SAQQgTCFsQFhSIIGkBELVCRa8Fqa1Fr2yuuty5XK23VX0Vs9Wpxa/W2FcV73W3rArQCitSlXhYVpBaEVtFAMCwhDASyzXLe3x+TGWYmM8msObM838/HD+bMzDnPmZy8z/vsSmuNIAiCkH9YzBZAEARBMAdRAIIgCHmKKABBEIQ8RRSAIAhCniIKQBAEIU+xmS1APPTt21dXVVWZLYYgCEJWsWHDhv1a64rw41mlAKqqqvjoo4/MFkMQBCGrUErtiHRcXECCIAh5iigAQRCEPEUUgCAIQp6SVTEAQRByA7fbza5du2htbTVblJyisLCQQYMGYbfbY3q/KABBELqdXbt20bNnT6qqqlBKmS1OTqC1pqGhgV27djFs2LCYPiMuIEEQup3W1lbKy8tl8U8hSinKy8vjsqpEAQiCYAoRF3+3G5qafP8KcROvQhUXkCAImUFDA+zYAUqB1jB0KJSXmy1VTiMWgCAI5uN2+xZ/wwCv1/fvjh1ptQRqamoYNWpU2s4fDz169DDlulllARxodvHix7vMFiOjKFk43WwRBCFuBl73GAe/8AZ+tnoMSrQRsiM1tEHTl1vx2tKzT23c9RVeVxsHv9iclvMH4/F4sNk6WW610akcvY9Nj6LKKgUgCEJuYlgUKmw4odK+434+qN/E+3s+YnL/8ZxeMSY11zW83HTbXXzw8SYG9DuGe+74CdfeejvvLvsjANu/3MEVN83j3WV/ZPSUc7lg5rm89e7fKCos5ImH7+fYqiHsbzjALT//BbvqdgNw7//7KRPGj+W+X/+WL3fWUrNzF4MqBzD1zEn8ZdVqGg8fYfeefVx8wfn89MbrQuQ50tTMZdfcyMHGRtxuN//vlhuYOf3r1NTUcN555zF58mTWrFnDwIEDWbp0KUVFRUndv7iAspymeavMFkEQkkZbFM097GjAUKDB93O7AvigfhMXrLqKezc9ygWrruKD+k0pue72mp1cefkc1q5cQq/Snnzy6VZKe/TgH59uA+D5V5Zw2UWzA+8v7dmDNSte46rLL+G2X94PwM9+cR/XXXE5f13yEk//9mFuun1+4P3//Gw7S559gsW//hUAG/++mWf++2HeX/4KS5a/ycefbAmRp7DAwbO/e4R3l/2RPz//FP/v3gfwj+397LPP+NGPfsSWLVvo3bs3r7zyStL3LxZADtA0b5W4goSsx+2w0tjbgsXQGBYVWPwB3t/zES6vCy8GLsPN+3s+SokVMHTQQL528ggAThl1MrW76rj8uxfy/MtLuOeOebz2+kpWv/pi4P3f/uZ5AFz0zfO4/R7fov7u/63nn59/EXjP4SNNHGlqBuC8aedQVFgYeO3sMybSp6w3AN88dyrrNmxk7OiRgde11vziwV+z5oMNWCwWdu/dx779DRT068mwYcMYM8Z3z+PGjaOmpibp+xcFkCOIEhByAW1ReC0dUxkn9x+Pw+rAZbhxWOxM7j8+JddzOByB/7darLR625g1Yzq/+s3vmTLxdMaMOjmwYENomqX//w1tsOqV5yksKOhw/uIwF014mqYi9Oc/LX2dhgYn7yz9A3a7ndFTzqWtrY0CoCDo/FarlZaWlrjvNxxxAeUQ4g4ScpXTK8awZPoT3H7K9SyZ/kTKYgCRKCwo4OtnTuLHd/6SSy+6IOS1V19fGfj3tLGnAHDO5IksevqFwHv87qNIvPN/a3EePERLayuvr/or1ePGhrzeePgIfcv7YLfb+dvaD6j9qi5FdxUZsQByDL8SEGtAyDVOrxiT1oU/mO/M/jdeX7War585KeT4wUONnDHzQgocDp58xOcCuv/O25g3/x7OmHkhXq+XiaeN4+Ff3hnxvKeOHsX3/+Nm6vbs5eILzg9x//ive8nV1zPpvG8x9msjOWF4bC0dEkX5AwzZwLEnj9b3PL/cbDGyClEEQiYy8LrHGD6wn9liROXRJ/6XxsOHueOWGwLHRk85l7eXvER5n7KEzvnCy0v4ePMWFt51R9yfjScNdOvWrZx00kkhx5RSG7TWHfxm4gLKccQtJAjx8b1rb+Kl15Zx7Q+/Z7YoaUcsgDxBLAEhk8h0CyDTEAsAaGtVNDqzSuSMQSwBQRDCyarVdPcOOzfO7M+alclVv+UrogQEQQgmqxSANsDVZmHRgjKxBBJElIAgCH6ychW12qC+zmq2GFmLKAFBECBLFYDXAxWV3q7fKAiCkGJ21O5i2oWXcuo5M7nihltxubJ3eE1WKQBlAUeBwdXznZSWGWaLk9WIFSAIiXHXrx7mun+/nI1vL6dXr1Ke/dOrZouUMFmlAAYMdfOb5XuYNCP5HhiCKAEhu9jfYGXjJ0Xsb0iN+/fehx/jd//zbODnXzzwG37/P891+hmtNe+t/YDZ5/nSqi+5cBbLV/01JfKYQVa1gigo1LLzTzHSRE7IBl5eVsqNtw3Cbte43YpH79vFRd9sTOqc3/vOt7j8P27mun+/HMMwePX1FSx9bjFnnv/tiO9/4uH7qSjvQ6+ePQPDXSr796duz76k5DCTrFIAQnoQJSBkMvsbrNx42yBaWi20tPqO3fCzQZw16Z/0LU88Fjhk0ED69O7FJ1u2sm9/A6NPPokhAyv5219ejvqZhgPOhK+XiYgCEABRAkLmsvMrB3a7Diz+ADabZudXDvqWJ+cOvvzii3jhlaXsq9/PZd++gMNHmpg55wcR3/vEw/dz4nHHcujw4cCIx7o9e6jsf0xSMpiJKAAhgHQSFTKRIQNduN2hffM9HsWQga6kz33+N6byX4/8N26PmyceuR+r1dqpBQBw5oTTWLpiFRd98zxefHUZ5007J2k5zMK0ILBSarBS6m2l1KdKqS1KqZvMkkUIRYLDQibRt9zLo/ftoqjQoGcPL0WFBo/etysp948fh8PO5AmnccHMc7FaYwsu3/WTm/ntU89w6jkzcR48yOXfuTBpOczCTAvAA/xYa71RKdUT2KCUWqW1/tREmYR2xCXkw9HqobjJTXOJHVehGMxmcdE3Gzlr0j/Z+ZWDIQNdKVn8AQzD4KNNn/C/jz4Y82eqhgxm9Wsvdv3GLMA0C0BrvVtrvbH9/w8DW4GBZskjdCTfLYHKmkNMW7adiW/XMm3Zdip3HDJbpLymb7mXU0e3pGzx3/bZdk79+kzOmlTN8GFDU3LObCMjtjRKqSpgLLA+wmtXA1cD9O0v+qG7yVdLwNHqYcwHe7B5NXh9LdPHrN/D/n4lYgnkCCOOH86md1aaLYapmF4IppTqAbwC/KfWukNir9Z6kdZ6vNZ6fM+yPt0voJCXlkBxkxsdNpxcWxTFTdlb9i8I4ZiqAJRSdnyL//Na6+ytp84D8k0JNJfYUUbosCRlaJpL7CZJJAipx8wsIAUsBrZqrR8ySw4hdvJJCbgKbWyq7o/HqnDbLXisik3V/cX9I+QUZj7NZwCXA/9QSm1qP3a71lpmPmYw+RQTqBvai/39SiQLSMhZzMwCel9rrbTWo7XWY9r/k8U/C8g3S+BgeZEs/kKARc+8wKnnzKRs+NeyvjWE6UFgITvJJyUgCMFMGDeWJc8+weCBlWaLkjSiAISEESWQHThaPfRuaMHR6jFblKRQDQewfrIZ1XAgJedLpB00wOiRJzFkUG6kpItdm0YanRbq66xUVHqljbVgCpU1hxjzwR60RaEMzabq/tQN7WW2WHFjX7ac4tvmg90Gbg/N9y3A/c2ZSZ0zkXbQI44fntQ1Mw1RAGlizYoiFt1dhtUOXjdcPd+Zk4Ns8ikonG3kSjGbajhA8W3zUa2t0N4RtPhn82mcNAFdnnhtUCLtoHON7HkKsohGp4VFd5fharNAm+/YogVljKpuy0lLQLqIZiaBYjbv0XoGfzFbNikAy1d1vp1/UDtobDYsX9XhTUIBQPztoMUCELqkvs6K1U5g8Qew2nzHc1EB+BFrILPIlWI2Y2AluMPiFx6P73iSJNIOOpeQIHAaqKj04g3rGOD1+I7nOvkUGM704GquFLPp8j4037cAXViI7tEDXVjo+znJ3T8k1g768f99npFnTKVuz14m/9tF3Hjb/KTlMIvsehKyhNIyg6vnO1m0oAyrzbf4Xz3fmdO7/2DywRLIluBqrhSzub85k8ZJE7B8VYcxsDIliz8k1g76mh9exjU/vCwl1zeb7HwasoBJM1oYVd2Wt1lAuawEsi246iq0ZaRc8aLL+yTt8w9m22fbmXPVjzj/G1OlHbSQekrLjLxb+IPJVSWQK8HVfEfaQUsMQEgzuRgTyJXgqqloA6111+8T4iLe71QUgJB2ck0J5Epw1Uxc9bU0trpFCaQQrTUNDQ0UFhbG/Bl5YoVuIdfcQbkSXDWLhuWPA9ewv2IwKNmHdkVxW2wZSoWFhQwaNCjm88pTK3QbuVYwlivBVTMwWhqpf2Wh2WJkDbNe2JqW84rqFbqdXHMJCUK2IgogA2l0Wti+xU6jM3d/PaIEBMF8cneFyVLWrCjixpn9ufe6Cm6c2Z81K4vMFiltiBIQBHMRBZBBBDeRazliwdVmYdGCMrEEBEFIC7m7smQhgSZyQfibyOUyogQEwRxEAWQQ0kROEITuRBRABuFvIucoMCgqMXAUGHnXRE4QhO5DkpgzDGkiZ37B2KHWMvY1DeSYkq/oVeg0VRZBSCeiADIQaSJnnhJ4t2Ymj33wS2wWDx7Dxg3VdzBl6Iq0XtPR6pGKYsEUxAUkZCRmuIMOtZbx2Ae/xOUtotndE5e3iEfX38Oh1rK0XbOy5hDTlm1n4tu1TFu2ncodh9J2LUEIRxSAkLF0txLY1zQQmyV0upfN4mFf08C0XC94roDdbWDzasas35OxE8aE3EMUgJDRdKcSOKbkKzxGqAvGY9g4puSrtFwvMFcgCP9cAUHoDkQBCBlPdymBXoVObqi+A4e1hWL7YRzWFm6oviNtgWCZKyCYjUSchKyguwLDU4au4JR+67olC8g/V2DM+tDZwhIIFroLedKErKG7lECvQme3pX/KXAHBTMQFJGQVuVgs5iq0cbC8SBZ/odsRBSBkHZmsBBytHno3tKQ0kycd5xQEEBeQkCIanZZurV7OhIrhcCprDjHmg1B/ft3QXhl3TkHwIwpASJo1K4pYdHcZVjt43XD1fCeTZrSk5NwFzgZK6mppqhxMW1l5yGuZNGIyOKcfry+zZ8z6PezvV5Kwaycd5xSEYMQFJARIZBJZOmcYDFmxhFkzJ3DOdZcya+YEhqxcGvF9meASSkdOf7RzljpbxSUkpATZRghA4rv4wAyDtqPH/DMMknEFFTgbqL57Hra2VmhrBaB6wa3srZ7cwRIA811C6cjpj3ROi9fg9Pd2oa0WcQkJSSMWgJDULj5dMwxK6mox7KGLp2GzU1JXG/UzZloC/px+j1XhtlvwWFXSOf3h5/RaFWiwGUjrCCEliAUgJLWL988wWLSgDKvNt/inYoZBU+VgLO5QzWLxuGmqHJzUedNJOnL6g89pd3kZ/391WN1Hv1u/m0liAkIimPrUKKWeAs4H9mmtR5kpSz6T7C4+HTMM2srKWT//AaoX3Iphs2PxuFk//4GI7p9gzHYFuQptKV+M/ed0tHqkdYSQUsx2Af0vMMNkGfKeVEwiKy0zGD7SndIU0J0zZrNs+Tre/v0LLFu+jp0zZsf0uUwICqeDdLiZhPzG1CdHa/2eUqrKTBkEH5k6iaytrLzLXX8kzLYE0oW0jhBSidkWQJcopa5WSn2klProsPOA2eLkNMns4hNJIU03sVoC2VZpK60jhFSR8U+Q1noRsAjg2JNH6y7eLphAOgvBkqHRaWH7zHcY+uq3ojZ3k0pbIZ/JnO2akFUUOBvos2UTri+daSsES4Y1K4q4cWZ/7r2ugiuXv897O87r8J7unMh1qLWMzxpGpXW8pCDES8ZbAELmMWTFEqrvnodht6NcblarxTzLpYHXU1EIlgzBdQ3+1NZHNzzAKf3WhVgCgUpb71HDMh1plWYMmheEWDB1m6aUehFYC5yolNqllJprpjxC1wRX6DqOHMbuauX3bVfSl/rAe1JRCJYMgbqGIPpZ63GNuTNkd98dE7nMGDQvCLFidhbQJWZeX4ifQIVue3sGAFVg43jjC5oc5SkrBIuFRqeFpm1OqtgBIwYGsoXC6xrm8CKLm+di/ZUNq8fNpnFl1A3t1S0TufyD5l1B+tA/aL67hs4IQjTEBSTERaQKXTturnmpmNrm+m5LIV2zooiDd67kce9VuLFTZHfz4YKF7JwxO6Q6uZ+1nsXNcymmBZp8nx2zwRnoqJnutMruHjQvCPEgQWAhLvwVup6CQlwlPfEUFLJs7sO09i5PeSFYNBqdFl5d4OFx71UU00IvGnG4W6hecCsFzgbAV9fwm+V7mP/Tj7GWhC7Ahs2OuvDhQJpoOtMqu3vQvJB9ab1mIhaAEDc7Z8zm8xOnsO2Fep7+80gOPlOBd3H60z/9Q2eaGhXHWr/EhcO3s2/HY/E1i/O7gkrLDAomD8B6b/SeQt0xU6A7B83nO5LWGx+iAIS4WbOiiMcXVOJ2KUCBy3d80YIyRlW3pcUKeOvlYp59oAyrXeP1QB+3xuG/cDs2o2OzuFh7CqW7crg7B83nKzJAJ37kWxHiwp9i6XZ19B6mK/1z9cvFPHVvGaDalQ402I/hKvUkT3ivPBoDmL8wYtuInTNms7d6ctTJYn5ytX1EvtBdab25hHwrQlxEah3tJx3pn41OC08v7A2ETsay2TVDFk7n93zYIQsoErH2FEq3EnC0eqSPT5rojrTeXEOeQCEuIrWOBo29QKcl/bO+zorNAZ4O7aoVVSPclJb1oo3RKb2mXwmkerFOhX9aFEh0uiOtN9eQb0aIi/ABMB43XHBlI1Mvak6L77+i0ovRIZlDc/m81Csbf5C5otLLkFE/ovrOG1MWTEyFf1oCnF0j3VLjQ74dIW66s3V0sMKxWH0K5wfzDjL1ouaUXie4oV2Zq54v9U9SGkxM1j8tAc7YScdQnlxFviUhIUrLjG7r9ZMKhePf3Q8u3kf/5h0hweDw3kEj2UkLdhxBKabJBhOT9U9LgFNIB/LkCFlBMgrHv7ufw4vc0nYVFNixaxfvznuQ+otmdQhs11CFndCgQ7LBxIB/et1uUAp0fP7pbA9wSuwiM5FKYCGn8e/uS9sa+F2br3K4uK0Ru6uVSffcyicvN3UIbO+ngqutT4ZUO6ckmKhBKV82k//fWMnmcZCVNYeYtmw7E9+uZdqy7VTuOGS2SEI7mf/0CEIS+Hf3VW01HSqH3dh5d+FBqqb2DAlsez1QNv9cllWvC6kdSDQ99FBrGYedfZn5wetYg1w48frwszHAKbGLzEZ+A0JKCM6gSSY2UOBsoPe2zSjAOWJUh9z9eK/j393XUNWhctiOm2ZbD2zvf8I5kwcwanlonKGNo7UDBc4GCmYuRL96c1wLl38WwOms5zLvu/SiMfBaIj78bAtwSuwis5HfgBA34YtwqkZCDlmxhAl33ozF68v7NOx21i14mJ0zZgOJjZ48mkVUzjXGkzzu9lUO23HzJHNZ03Kar030vb4WEf5rhcvlH4BjcR9tJ90VwbMAtjEy5XGFbCDbYxe5jtI6e8bsHnvyaH3P88vNFiOvCV+Ev3frQZ57oLcvg6YdR4HBb5bvCdmhd7VzL3A2MOu8amyu0BJjj6OQZSvWUU8FN87s3+V1/OcKb/vgv379hkN8+N9Omqw9WNt6WmgzuYIC3ntoMQeDLI8CZwOzZk7AFjT/wFNQyLLl67A9OafT7+qzhlHc+fZTNLt7AvBdXuQp5mK1urHiZVN1/4BLx22zYPcYWePaiYfKHYc6FGdJ/UJ8zHpha1KfV0pt0FqPDz+eW0+akFYijVp8dqGvQVtwa4jwnkCx7NxL6mrRVmuHa7a6rGx/ZR9tE/t3aEERqfdQ+G7dv6v3ZxENH1lMU3Ffttz/GW7sEKQArG1tnHnr1SjDCHwu0gAcw+brOnqgi7YR4bMA/sAl/M1yBk+ceT6Wshb67mli2rLtoDVWA7xWX2A4VQtkpmTeZGPsIl+QLCAhZiKNWrTaNJ5Q13pIT6BgpdHZ0PimysEob8c+QhYMHnlyNIXFukMLivDeQ+HjKm1trSEzAvzyPPtgbz73HtvRJQPYW5pDPhdpAE6kdtJ+gnvRR5oFcMmER/AM8J3PHxy1Gb5r27w6ZYPpMy3zJp0zF4TEEQUgxEykPkCG11eZ6ygwKCoxcBQYIT2BIisN3/Fg2srKWX/Xg3isNjSggTYcXMFinPYKWpsVV893Bq5jLzCYPbcx5ByB3XqwfO27dT9+efZTwRUsppkijlBCuCPU/7lIA3DC20n7lUCkRXfK0BU8OWsqd59zBU/OmhoYBh8IjkbAHyRNlODMG7vbSJlSEXKPTtWxUqoUqNBabw87Plpr/UlaJRMyjvA+QP75v5NmtHDa1NaIPv5ISiNa11D/oJk/fPcr3B4LmxjLfipweAwqKr0MH+lmVHUbq18uZsniUv7yTClLF5cGZOhqtx4uzx+4hNVM4zTbBpZZLgiJPwR/LpZ20p4rX2LMN8ZGTHeMNAsgUnDUT7JBUsm8EWIlqgWglLoY2Aa8opTaopQ6Lejl/023YJE42FbPvw5sMOPSQjv+UYu3/76e3yzfE/Dl+/zrHUdC+pVGNAshHMewMobffRrvFUyjqaQcR4HB5T8+SH2dNeA2WvpUKW5XR5dSLLv1cHkaC8oZfvdpbJh3F16HA3dxj4ifaysr58DIMVFbSpfU1WIU9Qg51tlOPriwy2PxWTweq4q7wCvS+EPJvBFiJWoWkFJqE3Ce1nq3Uup04BngNq31a0qpj7XWY7tRTp9MlUo7rivk9kkvcUKfcQD868AGtjas5aTyiYFjfjp7Tehe4s3f97//y612nnuwdyCAPHtuI395ppSWI0f3LkUlBrf/vp7hI32LbaQsoM7kGbXuVV/g2GbF4nKzYd4Cvvj29+K6v4jZQlbFW7OGd7qY+wO14VlAsQRwO+sOKpk3uUW6soA6UwD/0Fp/LejnAcBfgKeBH2qtT01KogRQlUpbrobL9ldw3tyN/OvABu5dMweP4cZmsXdQDJ29Jooh82l0WjqkftoLDNCETCSz2Q3+66V9DBwWv4+7szTPYOURi1IZsnLp0dGTrUcSXnRjafvsaPUwbdl2n8vJL3eYwsmULCAhecxIAz2slBru9/+3WwJnA0uAkUlJkwQ2rRjVXEzJwuls79OAp28rhgKP18v2165i7AHfH+f2Pg14KtwYePEYsLVhLSf0GZewYhClkVpiWVAjTR+z2eD8HzSydHEpWoPbpbBY4I5Lj0moAK2zNE+/XNFSS8MJjxV0VScQiVhbJ8Ti58+2qmGh++ns6biWsDl8WuvDSqkZwG1plSoKfTw2flo7kBGtRQCMai7GphUedEAx+BnVXIzN2I9HgU0bnLryJUqWLI2oNE4Qa6JbiXVBjRZAnnpRM6dPbeX2S/oBCleb7zFNZCh9V4Hj4NRSv5KoXnAre6snR1RcwaMn2xIYLxlrAFf8/EIq6CwNdAlwoVIqkK+nlOoH/A8wK81yRaTMawss/gAjWov4Re0QLttfwS9qh8T0ml9pWHSYNfHaVXi8rT6LwdvK9teuomTh9KOvGX5rws3WhrXAUcXwp60PcO+aOSEB6n8d2MDSzx6LGLTu7LVcJ5ZcfT+dBZBbmxU2R+gCGCm9NJxGp4XtW+yBgHJXgeNYUktTSawLezZ3BxUyh86elnHAfwGblFI3AV8DbgF+BXy/G2SLiRGtRSELf1ev+RXD5uJmRjUXZ4U1kUtWRiSXi9dqx/n+bjyTKzrs3qMNg4lkHXjc0NRoodFpCbwvONi7eV1BxIrkSGmefheVu7gkooWwp3gotVvsXQa1Yx0yH+yvj3WurVTYCsnSZS+g9sX/YaAOmKC13tUdgkXiuPIi/dC5VWk7/7bClg6KobPXthW28PPBO/Eon9LwWxov92ng+b71GAos2he0/nZ7bOLlPg08X3EAAy8WrHznpFuZffz1URVDZwoDoiuHWHzsZhAp6NpMESNKvmSvpyIuP/6alUWBmgRXKyiLwl6gA4s7msCC73GD4QGvt+teQuEuqu0XzGH4kpd8wV2Pm2dn/4Zrl86NqyldZ0ogUtBXFnYhmG4PAiulegP3A9XADGAmsEIpdZPW+q9JSZOhZJI14Q9ab21YG+R+OhrMhujWxJAVSzAW/5h3q+CsGrBc+VDAx262NeF3uVQvuBWv1Y632cMVLKa2qR8Qnx/fbx3UbLPx4C19cbcpPG5fPODxu8pAgTuobxFEdhkFXyuSz3/4kpdY+cJy7M1N7CkeyrWXjgzphxSLzNEsgWhB37dmDedgeeRnMdWkM1tIMpEym85+IxuB3wI/0lp7gDeVUmOA3yqldmitL+kOATOdeBRDZ6+FKw2/Yji1sIXXBhsdFAYQ0QVVXdcLY8PnnHs5uKzgmAhvPHkLBdWT+YeuSciaSDV+l4vz/d0s+NXYwOIPkRflzigtMygp1djs4A7KFrJYCEth6EikiuRoWUH25iYOjBxD7RZ7TE3pIuFvGRGsCLqjavdQaxn7mgZyTMlXHSqSY0k5TZR0nltIDZ09YVPC3T1a603AJKXUVWmVKgdIlTXRmTKJZGkUN7l5d5jCZdV4LeDS8O5gN1Pun83245ujps1+PPe+uGMTm3ZsZGPtek4dXM2YofGVhdRTQU3fSva4Q91T0dpEdFZIFrFHkUEHBWCzg1IGNvvRNhbh5+oqK6iwWONxhZ44mszR5CfIGugs6NvZwh0r/oE0NosHj2Hjhuo7Av2I0jmtSyaBZQdRfxOd+fq11k+kR5z8JprS6Ox4uHJoLvFw1jaNY7Jv8XcYcNaXmqZx9k5dU9tfuyqicthW2MK9/jiH9WgV9iuvb+aV1jlgdfHWXgcXbXmVi2aOCpyvM2siuD201gqr1cBRGH1R7qqddLQeRUDgWJm7nv+88hOOmdqf2uZjogZvg11Ufp+/PyvILwfKt6A5Cnz/dtbaIqr87UogMCw+LOi7as+sqAt3rAQPpHG166dH19/DKf3W0avQmVbrQ/oRZQfym8hywpWDq9BG30EDeOPZ3bw7THHWl5qiQQNwFtoY0WqLy5oA2FzcjEfpdsXgS489tu44ljTMgLNdYPGCdrHkzQ+YveFubD//Y6eB6007NvK7VZvxVpwDuyYCYC/QfPueFRzu/T59B03Al4DmI9IMgkg+92jZQqOq26h4ZSmzFt+C8Ywdy+L2uoORsylwNlC2bTMaQobARMoKCpGjHcPQXVYgdya/3xLwZ/P0craigV3F/XnsjegLd6zsaxqIzeIJnAPAZvGwr2kgvQqdaa0lkDqF7EAUQA5SN7QXjn4lTGly0zTOjjNoxxWPNQGRFcO+poHYdk7C63WAdoHhwLZzEvsK/8HxnVRobyts4eFBu/BOAc5wwNOrYddE1JC1vHjo2xgH3SzfGaow6uusMGQt9HsPas6GXRMDPvc9+sMQK8M/9CWYCuqZtfiWDoVctiOHGb/wzoC7x7DaWPeLRwLB8uCCLr8c4b5/uwNamzsPNET6XHDMwB8c7runKeAvP827mzm8xDP8e+AzwQt3rIQPpAHwGDaOKfkKIKr1kYodejrPLaQO+W3kKIm0AYg1NnGo5Ct07Rm+BbzqHag5G717DMeMvRPo3JrwWrw+37x2+T67ayLeAavRRkeF0TRvFYd7rMd1sc/VhNenNLz11RzusZ5HYohZTNht7RjUtVoZv3A+1iBfv9XrofquH3eo8PX77mMZSBOJaNXMg4v30WfLDpoqB0dsJf07fsRyzmc/FUDowh0r/oE0j66/J8SVFKxEkqkl6CrDR+oUMh/5jeQJyaTjhSuGkIVl76gOC0tX1oRbgzYcFOw6DW1t4aKST3klgsIoWTid3X0aUOWtaIsG7cJ67CpuGPIndr/13FEro5NeT3d97XGmBi30awfBX4e3MqXWyplfEHJ89XEGJV+sov84Xw+fcN/92Rc08c6SkpA4Q1eZP+HxCY8H7j/zKS679IZAncGWudf7WkkfORz4nLJ6OV5vo9laGHHhjkSkoPGUoSs4pd+6ToPJiWwWYs3wSaYfkaSQpp+sGgqf7kKwXCVd6XjxZKn433uoYhs1vfdSdbAfvepHBD4brQgvuNjOali585/HM+VgCxsqPPxseF1MRXjz33Rx7GeHWDsIpv0A2mxQ4IHVT8PEXb7Ff+oPfGmzVlsBt0/6A/3Vab5OpBXrA1aOo76ae17YR2uz6rICODxrqdFpYfXLxbz/ZBufu4d1GEaPJmQgjceqePXccdR6qmL6fjvL9kk1XXUiTcXCLSmkochQeCEh0pmOF2nSVSQiLU7jh66A8s2B98QSmzh7u8H3V/0TbVFMMjS9vtGHd461dBqzGHvQwdAv9qOAd6t8i7yhfP/+dRhM2AVvtx/3WkAbHrY2rMXqmuCLPVw8/aj76Y+raG0+LjB3IFqm0yuvb2bJ6o1YvzoLvXMiF8xt5PSprSx9qpTR7g24cIQoAMPmYOsPrmXk4kdDWkkX9jrC8WymK7rK9kk10TJ8DGcRloYipn36EdpCwgu3pJB2H6Z+m+2dRX8NWIEntdb3mSlPLmJ2Ol4qFqcRrUWMPmhn2qrtIYvC9988QGXYwJVw99O4egL3f3YNOLy+9FgbiuIhlaw9W9G3tA0b+9FaB4rtBhx8D++AGb7Fvz3TyTtgNUNfnQcjo2c6bdqx0ZceO8XlC5I/vZo//XYCrz3RCz14DZ9WruDjmhbOCUqytnjcbL/oMrZfdFlI5lGsnUTDs336Us8ItnDY2ZdeA6J/x4nWGUTK8NFeCz9992nW6zOwYUC7LIks3GY/s/mEad9me5fR/wamA7uAD5VSy7TWn5olUzYQr3ltdjpeV6mIsRLPohBsTTSXeAL3P3GXz+3z12OheHAlw42e7B8A/enB3bXFfOTQ9KsfzgDdSq9CJxeXfMqLQZlOF5d86pO5k0ynfxTaYPBRpeEPdHv6rYVLptNsdTHda2XV0w5O3VVISYGL9fMf4B+6xmdNDJjICe1B6KZ5q6h9dHJArtPb5QonONtnDi+ymLm4vXZK/naETdX9Iu7Ak3EZdczwgSuMJynUng7WTSILt9nPbD5hpjo9Hfhca/0FgFLqJWA2IAogCon4Rc1Ox+sqFTFWuloUIilG/w63eCxM+ni7L8Vyt8Y+pD91Rs+Qc+3ddhFL2xfERe0L4pyhKziuZjgfOWC8C8b3/nvg/dEynca7YEWQ0qDmbN8Hqt4JWBNeDbOH/ZifTJtO8WkD2Fu5nUfWfDdiI8BfDt2Lx3BD33pszy7npkHLmDJ0RWjMBLih+g5eXPefLDbm8vdBLbxT1cLZNXBahB14Kqyy4Ayfza7RvPZ/F1LsbsWBK+LvKB5rw+xnNp8w8xsdCAQ3Vd+Fr/FcCEqpq4GrASqK8/cBSMYvamY6XiypiLHQ2aIQSTG+qC8J2eH+7NSbmVq2MuL9d7Ygji90Mr61ozzRMp3GGx4u2TiLPzaNxPvF1ECxGzVng9eBRbVgRTPX8zFrnp7MlueK+WrSJoyvu9FhDf827VrnW/zbrQnPkLU8uvYeioet4v6wLrRThq7g6463+PALF+f5+0B54Y3nCezA/Uqj1+4BKbHK/Bk+PVv34zFs7KeCK1jMU8zFjZ0Sq88CSaSqWVJIu4eM/1a11ouAReDLAjJZHNNI1i9q5njAWFIRYyHSohBZMe5ltr4Jl3F0Qb9v48McN+sTKqind0NLyKKSqJsqWuB6Tu+/c17hTt5w9uCPdWOxWz14do/hoo2zKCtbziV/O8wZtauAVbg8Dv7ti9tZfaYDi92FzWLnpHKf0qhwTwbvr0OsCWUv4L0xs/A0bfV1iFUWNs6Yw4glS7GUtfB+lTcQ0HZpeG+IZlyJPbR1efm7GB9tgJ1n+wQetJa2Y1dxqGIbhCU2ddYi3U+wkv+z5XyO927jJyN/xonHfUg9FTy2LDFrQ0Zaph8zv92vgMFBPw9qPyZEINv9orFmDHVF+KIQSTF6sXCs+oI6hgSO2Sweyj73Mu3T7R1caKlyUwXTq9DJxaMWce5xfwoovgrqmfbSYWxBi2wBLpbtup9jn3uWr/14I5NPPj2QUTTu2LH8z3+twlN5tAq6Dc37v5uJ5fsPtd+XT2E0zbueJqDv8odxtDwU6APV9xu34Jx5Mxs/ewzP1gfalYbi9KtWs/GXU2DIWlwXT8ewu7jfYucXX/brct6F/7VgxRBJybuwsa8hNTEgIT2YqQA+BI5XSg3Dt/DPAS41UZ6MRvyikYmkGK0YfKGPDTnW2+tkxqcfRXShpcpNFYlgxVfc4MaDDRuhvYMMLBxbewzfHvmjkNqC0jKDa68cweN3TcQd6ECq8H5xBrZn3mLWnX9hzKAJISmopTNv5uc7TuFftas5YfBUSod+HYCTyidis9jxGD6lMWPieH64fA8vb1nO6gOugPtp44w5DD7+eoAwpWEJvPavAxu4971vdVAMvQqd7O5dx6ogxRCiXAethap3cO2cmJRyTTX5XHBm2t1qrT1KqeuBN/ClgT6ltd5iljypJF0PVKr8otn+wIfL31Ex9uMSHglZ0H9y8s/Q2wikJ0KoCy0VbqquAp17bBUYxl4IUwBWvEwf9zwDnnw3MDPA36juW72g392n8ssFJ9LWcrTvkH3vRMZZj2N4n7A+E0DV0K9T1b7w+/maquK+Ab9gg30nxw+a2q40DCaffDrvrTmqGE5xnEyfLZtoqhzcQWn4XVNbG9bisVgiK4ZAauxhnzWBL0D9612z8Fw+0xcI14rduwbSK4o10Z3ke8GZqX/9WuvlwHIzZUg16X6gkvWLZvsDH03+cMU4hdAFvYJ61Kedu9CScVPFklZZ66niOst/sMi4Bge+hduFg9dPqmbcce8G3jdkxRIm3HVLoFHdWVYb23ia54MM5OA+RNFmJfiPT9j6Mmc/eGug9cT6+Seyc4bPajihzzhun/QSWxvWcs6/3Mydc03gff3mPwDtrwUXu3WqGIKm1/kVwzjg/E/+m6U1PkvDwBp4rfbRyVHdTBBdOaRCaUjBWRYEgbOJTH+gYpUvUy2EruQPlzV4QXeRPhdarGmVx5R8xUtqDss5n7N4m37s5T01mZ+M+BGO9vf4RlLe2qFR3VO2ufzVMZUD1goM42gfomizEvzH+9nquaVpHjZCu6EGN707oc84vqaqmPX9CR26pu5dvo4Tjg+d5xCsNGJRDABjBk1g+c6Or22cMSeimwnCRp4a+wPKIZ7YRDDhr0nBmSiAlJLpD1Qs8mWyhdBZC4LPHCd26bqJ14UWa+56rFlE/ljD3rWn8Li+Djd2ilQzn+ytCHzHJXW1aIu1wzUMZWGoUcMBa0VgtHG0WQNDT3QHjvdr2xmh9YSdkrrakK6n0UZhhr/Pzwl9xnUY9BNNMXT2WmdKI8SisBy1GuKNTUDkgPbo9vjR2kHwThW+uond8SdWpGJym1mYvyrlEJmeqRNLMVX4DvuUdfv4ovdACnsdiXjO7rQWorUguOq9v3DQWhZTjnmsLrR4KmUH22o4xbuJzxjRZfvmaf3+zDTLr7F5DaAFjFArRr96M8ro2GLacBt8zrG0eXwDaRYtKOOWhxoizhr4fPPRucU1VHUozgoecemnq1GYsRJJMXT2WmdKI5pyiDc2AWEBbTSbi5sZ0VrOM9/ow/WnNgTqJh7b2Ie+7c9HLNaEe/tUXl31VLc04UsHlq7fIsSKPyDpsSrcdgseq8qoTJ2u5AvssINoNkp4ZOVDvLfjvA7nq6w5xLRl25n4di3Tlm2ncsehbpbfwhX6SeqMITS7e+LyFvHo+ns41FqW1HWCXTpdnbey5hAXvrGB1WoaOxjK9y3/g8PaEjWLyPcdhx7zW2H+e1w//0G8djsa32bfY7VxbcHigHIB30IPkWcUHDfKHTjuL85qpoi24p54CgpZP/8B6qlg+xY7jU6fMP5RmJ6CAtxFxXgKCgKjMNPNCX3GMfv466Mqh++cdGvIvIdox/2KwYK1gzUR8pq1kOHfeoKmeat484dX0ma34LVAm93Kmz+8kqZ5qwIWw/N96/n54J1sKzxqQQW/9tK4pbgGbAo8J7/eNYvnSptD3p/JZMbKlENkegVjZ/JF2mHbcfOZMYINYT5ts+IdkVoQELQIpiLHPFaXTvB3YGvfZT/JVfzbub+NajHFYiX6R1L6x1XW9h/NHy4dGfIZrweqRngizkIeOCz0+Gue7zLsx6dy9kmf0VQ5mLfXDWLRzAgzlrUOuJcIFdE0olkUqXVBOQJtOGKJTQRbE1iO9nti0Fo8l8/kZVsLS5OITXQnmbU65QiZXsEYTT7/DvuUdftoNkqw4+YKfDvPYsvhkAXQzHhHeAuCYJIt4ILY+xdF+g6wavp76jlI5D/kWOs52srK2TPxLAAc+BbpV+/ycKylhi+MKi6cb6O0zIg6C7nj8RIOMCZq3GD8iXVU3z0vZCZBeLA4G0iVC6qz2ETwa0aEfk9aEaI0Ost0ijWgPSu1X1OAzF2l8hSzM3Dqhvbii94DeWTlQ3xmRPdpZ0K8I10FXLGeN9HvoCsr0d8G2l8TADD7yEs8qOfhxYZduVnPA+xkNkDEWcjRjkebUezZ/FVcQeBcIhlrwvPZ2fy5vhpriYG7bgpY7BjEnukUOTbRMdPpzNq1TBw8kVQjCiCDyJQMnMJeR5g04TU2rL+HYsvhiAtgplQmp6rPUCLnjfYd1FPBvobO5YnHSvzk5SZ+fu88HLQE3F2x7M4j1QdEm1FsGzUwJUHgdBKt3iFdxGRNHA/TJ+5pl2sYe3Ti1oTNavfFJvqM65Dp9E7NO2lRADISMkPoasyeGcSS3ma2xZIJBH8HiXS+jEbTvFU0Oi08PaOW5e5z6c3RIHtbcU/eefwFDowcE/Gz0eoDANasLOoQN5g0o4UhK5dSveBW31Qyj5v18x9g54zZCcmeajq7n2wg2vS4aK+FDxx654d/TUoBRBsJKQogQ+jd0MLEt2uxu4/ubNx2C2vPGczB8u4NDAmxEa4gD7WWceWy1bi8R39fDmsLT86ampBl0jRvFdu32Fl0LWxrCp0j7HYU8ucV6yJaAI1Oi2+ecdvRdCNHgcFvlu8J7Jyj7aYLnA0hU8kygVjuJxcJVgzzpyaniGUmcIaTCT71fCTRIp5IdQIDetSmtPNlycLpVFy5mr2e/iF99u24WTMveopmJD+/xQofv1/A2MltgdhApMWzraw8YxZ+P9HiFvV11pxWAJ25oFKF1AFkCJleQ5AqHK0eeje04Gj1xPTeit1H6Lv7SEzvj5d3a2Zy5bLV3Pn2U1y5bHXEWodIRKsTKLQ1dZo9dKi1jM8aRsVVp1BaZnD1fCevFXyXEcVfMsP2Jjf8+yds//oFUT8Tyc/f2qx4+ldl3DizP2tWZpdFGS1u4e+FJCRObq0uWU6m1xAkSzxB7sqaQ4xdtxtLu1FkWODjCQNSFhRPZixitDqBVk9J1OyhZGbw+lM6V79SzJLFp/H3P8FzL0T3g/uVxqIFZVisvsUfFK1NviI/f7uI1mbVbQHVZAi+n+C4RabLnQ3k1gqTA2RiDUEqAr3xFI45Wj2MWb8ba3B6vQFjU1holsyw+s7qBI4v39wheyjZGbyNTgs122wsWVyKu82COyh/f1R1W8SFcNKMFoae6GbNG4Usf66UtuajFd5aw+2X9MPm0FkTUI1W7yAkR2atNEJGELzg993TlJLU1HgKx4qb3KAU4eWoGlJWaJbMFLCu6gTC20ono2w++tHDPLZxIcoC7rbQNh2d+cHXrCji1QUehln+Sc/WYbRxTOA133AZFRgy05kiySSixS2ExBEFIIQQ4qbxGih8u+9k2z3EE+RuLrH7tqlhKP9rKSDZIrJ46g8SVTZHLYfIobpofvBGp4WDd67kM+9VuHDgwMUVLGZp8XfxuMFiAVeQMrHaoGabnQr2UcUOGDGwy0Bwd+fkC+lBFIAQIJKbJnwZTrTdQzyFY65CG5smDGDs2rAYQIqD4skWkcU6QCZRZRPJcgBNQZFGG9H94E3bnDzuvYpiWgKpo08xlzuvP5X+1b2449JjQt7valXsvOFNfm5c5WtRbXfz4YKFUWsAUpWTL0rEfEQBCAEi9rYJo6vU1M7iBfEEuf3v7eVsRQONZYVpiY2kalh9VySibCJZDnaH5uYHGqga4Y66aFaxAzd2CK4bwM6kodtpGzY6EFDV2ucOKvPW8wRXtSsLX7VxtErjaL2E4nUhZXthV64gaaBCgEhuGkOBN8bU1FjaQ7sKbRwsL4ppMXcV2qgf0IP9A3pkXGA8EXoVOjm+fHPMCsdvOTisLRSVGDgKDK65y8noiR0X20an5Wh75xEDKbKH5k0W2d0wYiDgC6je88I+fE41RRU7cAVmkvnwWHx9gMIJ5OQH4Y9FxEqwEmk5YsHVZmHRgrJAa+p0U+BsoM+WTRQ4G7rleplM9v9VCRFJJHMnmpsmll17po/DzFb8lsOOC18LuErCXScdd9MF9FuwkOoFt+Kx2LEZbj6cvzBkN9/arLA5NG6Xijg0xma42VM8lNot9ph6CcWTk29mYdeQFUuovnte0HzkzGl3YQbyl5mDJNNULpqbpqtFPNPHYWYzvQqdDB/pptFp4bUne7BkcSm29sX+8h8f5NkHe3d0ySz/FnuXT47a1iF4IfcPjfFXGhfZ3Tw/+9dce+nIDi6aVOTkm1XY5Zu3PK/j3OMsa3mdSuQvM8dIxU48kVoEaWWRXtasKOLxBWVHUzjbF/unF/bG5iDybnpk9LYO4Qv5q57vUnLpafzb+H9xuP9grrl0JO4ofv5kc/LNKuyKd+5xPiAKIMcwayfudx+NXb8Hjc+7nIutLMzgUGsZi+4sxh0hHdRnCYTWB8S6m+64kBfSxmhef6JHlzUHyebkm1HYlaq5x7mEBIFzDFN34hr83WUzrctsPD2IMg1/OmgkDC9cPs+Jo8AIBIrj2U2XlhkMH3k0o6jRaWHJ4lJ8KvwoHg8UFuuQOcLJEn7taIQEuJPg6NzjQlwlR+cj5+vuH8QCyDnMGtQScD0Z4K8eyJQgcKYM2kmUSOmgoLEX6IBv/vSvtya9m250Wvj4/QKsdnCHxIQ1p05p4Y5Lj0k4bTPRnP9Up4v65y1nWstrsxAFkIOY0VQuU4PAuZCdFFxIZi0swOOGC65sZOpFzSlzyfgXWouNQNM4Pza7ZuN7RVFjArGeO95FPFU1B+FkYstrs8iOv4AsxqyJWd3dVC5Tg8CZqpjiJVI6aKqItNCCprBYY3hh9txG/vJMaSDwDLGnbSaziOfrHIBwGp0WPvwQqqqgoiK1586ev4AsJNtdD/GQKTOCw8lUxZQI/nTQVBNpoXUUaL57/UEmnuvLmFm6uDTkM7EGmuNdxIMnklVUVuT9HAC/9VRUCC4XLF4Ml1ySuvNLEDhNBLse7G4Dm1czZv2erAxCxkrd0F68NWs4a88ZzFuzhmeEssuXQTvJECkv39WmeO6hMjavLwikbSYSaI4n53/IiiXMmjmBc667lFkzJzBq/WsJXzcXCLaeDh2ClhaYOxfq61N3DZkJnCZkxm9mkUvD65vmrer09UQCrm+9UsxT95QRnv1jLzB4tH32bsKB3ChD6IMpcDYwa+YEX5FWO56CQpYtX0c9FSHXNaOJnBmzkrdvsXPvdRW0HDm6Ty8thbfegtNOi+9cMhO4m8kl10MukImDdtJBogHXYSPcOAo1rtZQBWCxHHXXJBpojiXnv9MirZHlgc+Y0UTOrPYRkawnt9sXC0gV4gJKE+J6ELqbZJqsVVR60RHWdsNIjc+9q5z/WIq0zGgiF9w+wnHkMLa2VqoX3NotjeSCXW+lpVBU5IsBpDIQLKtRHMTrRsj1Gb/5Sqa6k2IJuEZzn5SW+TqN/u7nZXi9PivAZodrusnn7i/Sql5wK4bNjsXj7lCkZUZWkNntI/zW06k9KyULyEwSzejJF9dDvpAJmV0lC6dHjANEchmUues5uXEDOAfy9rpBnbpP/ItNzTYboDqdOZAOuirSMqOJXCa0jygtMzhtbHrOLS6gGMikjJ5sbmmQ7WTScxCJ8Gyd71mf50s9jPN/OodZMyfgvPONLt0npWUGoye6Is4c6A7ayso5MHJMxN11MtlIyciTy+0jZGsaA5lSTJQJu898JlOeg87w7+Kbtjm58ZYrfVk17RvYRVzJG0xnPz4/QjYWVZnRRC6X20dkxlOb4WRCRk8utDTIdjLhOYiF0jKDqtIvO/iu3dipoiagALK1qCrZtheJkKvtI8QFFAOZkNET2H0G4d99Ct1DJjwHsRLJd11kd1PnGJKXRVVCZEx5cpVS3wHuAk4CTtdaf2SGHPFgdkZPtuw+cx2znwM/0QLBfiJl1Xw4fyF3Vnupr6vv1iIqIXMxa+uyGbgQeNyk60ekq/Q+MzN6MrXXTj6SLZldkXzXpXS/+0TIXEx5irXWWwGUUl29tdvIhgBrpuw+hewh13zXZrRkyGUyPgaglLpaKfWRUuqjxjSl22V6el8wrkIbB8uLZPEXKFk43WwRupXwZnFDVi41W6SsJ20KQCn1llJqc4T/4mqiobVepLUer7UeX5qmRU8CrIKQ2ZjZkiGXSds2Ums9LV3nTjUSYBWEzMbslgy5Ssa7gLqDbErvE4R8JBNaMuQipigApdS3lFK7gInA60qpN8yQI5hMHGYiCF1hZhygwNlAny2busUNk+stGczCrCyg14DXzLh2Z2RLep8gmI0ZPfJzuSWDWchqJwhCXAQHZP0++eoFt7K3enLaF+VcS2s1G4kBZBHSCVTIBAIB2SD8AVkhuxALIEuItVAtU4eVCOmjq7YQqSbegKwZM3yF2JAVIguItRNoNlQzC9lPLNO7/Jgxw1eIHVEAWUAsfeilXbTQncQSkA2e4esf47hoQRmjqs0ZNiN0RFaGLCCWQrVsGFYi5BZdBWTNmOEbC+KSOooEgbOAWArVpJo5v8nEvkBmzPDtijUrirhxZn/uva6CG2f2Z83KItNkyQRka5gldNUJVNpFC5mGf4bvogVlWG2+xd/MITTRXFJDT3TT2qzy0iKQ1SGL6KpQTdpFC5mGGTN8oxHJJQVw25xjsBfkZ5BaXEA5hrSLzl8y0Q0EPktg+Ei36bvrSC4pV5vC47bQcsSCq83CogVlNDrzZ1nMnzvtAimyEoTcprTM4Hu3HsTu0BQWG9gdGntBaNzMH6TOF2SbiOTPC0I+sGZFEc890BurXeNxwcX/cZCXfx/6d252kLq7yXsLIJumgQlCrtHotLB9iz3tbpfgAHBrkwWP28LLv+/F9249iKPAoKjEwFFgmBqkNoO8twAkf17IJbq7LUQydGeVcLSahGEj3Pxm+Z6MCFKbQd6vcJI/LwjdT3dUCQcXfHVWk1BaZuTdwu8n711AMg1MELqfwI48iFQGYMMLvjavL+Dq+c68dvdEQlY5JH9eELqbdFYJR7MufrN8T167eyKR9xaAH8mfF4Tuw18lnI4deWfWRabUJGQKstoJQo6RLYHgdFUJZ2IPokxFLABBEEwjHTvydFoXuYZYAIIg5ByZ1IMokxELQBBykEztC9SdiL+/a0QBCIIg5CmiAARBEPIUUQCCIAhport6HSVKZkolCELSSBzAXLJh/KQoAEEQhBQTXI2cycNmMksaQRCEHCDdvY5ShSgAQRCEFJMt1ch5rwBkFKSQy0gcwByypRo5ryuBZRSkIAjpIhuqkfNWAQSPgvRPAxuzfg/7+5VIR1BBEFJCpg+byVsXUGAUZBD+UZCCIAj5QN4qABkFKeQLEgcQopF3CsAf9AVkFKQgCHlNXq12kYK+b80aLqMgBUHIS/LGAggO+trdBjavZsz6PQAyClLIecQNJERCaa27fleGoJSqB3Yk8tkeUHw8nGABaz1QARjg/Qz+dQSaUypo99IX2G+2ECkkl+4nl+4Fcut+8u1ehmqtK8IPZpUCSBVKqY+01uPNliMV5NK9QG7dTy7dC+TW/ci9+MgbF5AgCIIQiigAQRCEPCVfFcAiswVIIbl0L5Bb95NL9wK5dT9yL+RpDEAQBEHIXwtAEAQh7xEFIAiCkKfkrQJQSv1CKfWJUmqTUupNpVSl2TIlilJqoVJqW/v9vKaU6m22TMmglPqOUmqLUspQSmVlqp5SaoZS6p9Kqc+VUj8zW55kUEo9pZTap5TabLYsyaKUGqyUelsp9Wn7M3aT2TIlilKqUCn1gVLq7+33siDuc+RrDEApVaq1bmz//xuBk7XW15osVkIopb4B/FVr7VFK3Q+gtf6pyWIljFLqJMAAHgdu1Vp/ZLJIcaGUsgL/AqYDu4APgUu01p+aKliCKKWmAEeAZ7TWo8yWJxmUUgOAAVrrjUqpnsAG4IJs/N0opRRQorU+opSyA+8DN2mt18V6jry1APyLfzslQNZqQq31m1pr/0izdcAgM+VJFq31Vq31P82WIwlOBz7XWn+htXYBLwGzTZYpYbTW7wEHzJYjFWitd2utN7b//2FgKzDQXKkSQ/s40v6jvf2/uNaxvFUAAEqpe5RStcBlwJ1my5MirgBWmC1EnjMQqA36eRdZusjkMkqpKmAssN5kURJGKWVVSm0C9gGrtNZx3UtOKwCl1FtKqc0R/psNoLW+Q2s9GHgeuN5caTunq3tpf88dgAff/WQ0sdyPIKQLpVQP4BXgP8O8AVmF1tqrtR6Dz+o/XSkVl4sup1tgaq2nxfjW54HlwPw0ipMUXd2LUuqHwPnAVJ0FgZ04fjfZyFfA4KCfB7UfEzKAdn/5K8DzWutXzZYnFWitDyql3gZmADEH63PaAugMpdTxQT/OBraZJUuyKKVmAD8BZmmts7mzaa7wIXC8UmqYUsoBzAGWmSyTQCBwuhjYqrV+yGx5kkEpVeHP+FNKFeFLOohrHcvnLKBXgBPxZZvsAK7VWmflLk0p9TlQADS0H1qXrRlNAEqpbwGP4uvafRDYpLU+11Sh4kQpNRN4BLACT2mt7zFXosRRSr0InI2v7fBeYL7WerGpQiWIUmoy8DfgH/j+9gFu11ovN0+qxFBKjQaexveMWYA/aq3vjusc+aoABEEQ8p28dQEJgiDkO6IABEEQ8hRRAIIgCHmKKABBEIQ8RRSAIAhCniIKQBDioL2b5JdKqT7tP5e1/1yllFqplDqolPqL2XIKQiyIAhCEONBa1wK/A+5rP3QfsEhrXQMsBC43STRBiBtRAIIQPw8DE5RS/wlMBh4A0FqvBg6bKJcgxEVO9wIShHSgtXYrpeYBK4FvaK3dZsskCIkgFoAgJMZ5wG4gqwekCPmNKABBiBOl1Bh8jbcmADe3T5kShKxDFIAgxEF7N8nf4esjvxNf4PcBc6UShMQQBSAI8XEVsFNrvar9598CJymlzlJK/Q34EzBVKbVLKZVVHUyF/EO6gQqCIOQpYgEIgiDkKaIABEEQ8hRRAIIgCHmKKABBEIQ8RRSAIAhCniIKQBAEIU8RBSAIgpCn/H/azeb1KuLThwAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEJCAYAAACKWmBmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/c0lEQVR4nO2deZhT5dm47yfLbKzDMIKsg2g/VKpYVBaXqmAFPrcWbdXKZ1ss1q+t2lbbqr8W0epnq9a61FoU61K31gWpRVvca0VUkFoQqqAjDIsMw8AgzJLl/f2RZMhkkkyWk5yT5Lmvi4tJcnLO+55z8jznWV8xxqAoiqIoiXDZPQBFURTF2aiiUBRFUZKiikJRFEVJiioKRVEUJSmqKBRFUZSkqKJQFEVRkmKbohCRChF5S0T+JSKrRWRenG3KReRxEVknIstEpM6GoSqKopQ0dloU7cBJxpjDgXHANBGZGLPNbKDZGHMgcCvwy/wOUVEURfHYdWATqvT7LPzSG/4XW/13BnBN+O8ngDtFREySKsGBAweauro6awerKIpS5Cxfvny7MaY23me2KQoAEXEDy4EDgd8aY5bFbDIU2AhgjPGLyC6gBtges585wByAESNG8M477+R66IqiKEWFiHyS6DNbg9nGmIAxZhwwDDhaRMZmuJ/5xpgjjTFH1tbGVYiKoihKhjgi68kYsxN4GZgW89EmYDiAiHiAfkBTXgenKIpS4tiZ9VQrIv3Df1cCJwNrYzZbBFwQ/vss4KVk8QlFURTFeuyMUewPPBCOU7iAPxljnhWRa4F3jDGLgAXAQyKyDtgBnGPfcBVFyTc+n4+Ghgba2trsHkrRUFFRwbBhw/B6vSl/x86sp/eAI+K8//Oov9uAs/M5LkVRnENDQwN9+vShrq4OEbF7OAWPMYampiYaGhoYNWpUyt9zRIxCURQlHm1tbdTU1KiSsAgRoaamJm0LTRVFqdPYCG+/HfpfURyIKglryeR8qqIoZR59FEaOhJNPDv3/6KN2j0hRFAeiiqJUaWyE2bOhtRV27Qr9P3u2WhaKEkN9fT1jx2ZU4mU5vXv3tuW4tlZm54KdH69m0XkH2z0Mx9O/qZVJ/nai8x58/naWnn8UO2sqbRuXokQz9OI72flRwNYxtDRsItDRzs6PVuX8WH6/H48niVg2wYTj6H9A7pRZ0SkKJTX29vIiwa4lKRI07O2VespcOqytaGVV1V7G7q1iTJsqIiV3vNW4kte3vsOxg4/k6NpxluwzGAxw6ZXX8Na7K9l/0H5cf/WP+c7lV/Hqoj8BsP7jT/jWpVfw6qI/cdjxp3DmjFN44dV/UFlRwT23/pID6kawvWkHP/zZdTRs3gLADf/vJ0w88ghuvO0uPt6wkfoNDQwbsj9TjpvMs0tepGX3Z2zZuo2vnnkqP7nk4i7j+WzPXr5+0SXsbGnB5/Px/374fc67aCz19fVMnz6dY489ljfeeIOhQ4fyzDPPUFmZ3W9OXU8lSkeFh5UTBuN3Cz6vC79bWDlhMB0V1j87rK1o5WfDN/DwwEZ+NnwDaytaLT+GokBISZy55NvcsPIOzlzybd5qXGnJftfXb+DCWeew9PmF9Ovbh/feX0Pf3r359/uhGuGHn1zI12ee0bl93z69eeO5p/n2rHO58hehptc/ve5GLv7WLF5a+BgP3HUrl141t3P7/3y4noUP3cOC234FwIp/reLB397K64ufZOHiv/Pue6u7jKeivIyHfvcbXl30J/7y8H38vxtuJlKL/OGHH/Ld736X1atX079/f5588sms568WRQmzeWQ/tg/qRdUeH3t7eXOiJABWVe3FL4aggB/Dqqq9alUoOeH1re/QEeggQJCOoI/Xt75jiVUxcthQPn/IGAAOH3sIGxs2M+trX+HhJxZy/dVX8PRfn+fFp/Ylg5x12nQAZp42nauuDwn/V/+5jP+s+6hzm92f7eGzPXsBmD71RCorKjo/O+GYSQyo7g/AaadM4c3lKzjisEM7PzfGcN0tt/HGW8txuVxs+XQbn376KQCjRo1i3LjQnMePH099fX3W81dFUWKUtfm7KIbIv1wydm8VHiP4MXiMMHZvVU6Pp5Quxw4+kjJ3GR1BH2UuL8cOPtKS/ZaVlXX+7Xa5aQu0c/q0k/nV7Xdz/KSjGTf2kE7BDl1TUCN/B02QJU8+TEV5ebf9V8W4hmJTWIWur//8zF9pamrmlWcex+v1ctjxp3TWRpRH7d/tdtPamr0Fr66nEmJI/S6mLlrPpJc3MnXReoZ8sisvxx3TVsl1G0fw9e21XLdxhFoTSs44unYcC0++h6sO/x4LT77HshhFPCrKyznpuMn86Oe/4LyZZ3b57Km/Pt/5/1FHHA7AicdOYv4Dj3RuE3FbxeOVfy6leecuWtva+OuSl5gwvmsTi5bdnzGwZgBer5d/LH2LjZs2WzSr+KhFUSKUtfkZ99ZWPAEDgZAvc9yyrWwf1CvnFgWElIUqCCUfHF07LqcKIpqzz/hv/rrkRU46bnKX93fuauGYGV+hvKyMe38Tcj398udXcsXc6zlmxlcIBAJMOmo8t/7i5/F2yxcOG8v//O8P2Lz1U7565qld3E6R454753tMnv5ljvj8oXxudOrtODJBiq0Z64E1lebXp9TZPQzH0b+plUkvb8TrC3a+5/O6WHri8Lylw2rmk5IuQy++k9FDB9k9jITccc/9tOzezdU//H7ne4cdfwovL3yMmgHVGe3zkScW8u6q1dx0zdVpfS+d9Ng1a9Zw8MFdywhEZLkxJq6vTi2KEiHf6bCxRDKf/BKKU6gLSil0zv/OpXy8YSOL/rjA7qHkHFUUJUIkHXbcsq0YlyBBk7N02Hho5pNSbPzx7tvivv/ea3/Lar/nnXUm5511Zlb7sBpVFCVEvtJh46GZT4pSuKiiKDHykQ4bj0jmk8YoFKXwsHMp1OEi8rKIvC8iq0Xk0jjbnCAiu0RkZfhf/BQBpSAY01bJWTtqVEkoSoFhp0XhB35kjFkhIn2A5SKyxBjzfsx2/zDGnGrD+JQYYov1MkWznxSlsLBzKdQtwJbw37tFZA0wFIhVFIoDGFK/i3FvdQ2Ebx7ZL+39aPaTUgp8srGB2Zf+mB3NOxk39hDuvuX/KCvLT4ZhLnBEZbaI1BFaP3tZnI8nici/ROQ5ETk0zueIyBwReUdE3mlp8+dyqCVJdLGe1xfEEzCMW7aVsgzOdZfsJwllPylKsXHNr27l4m/OYsXLi+nXry8P/fkpu4eUFbYrChHpDTwJXGaMaYn5eAUw0hhzOHAHsDDePowx840xRxpjjuxrQ6C22Kna48O4uvaaMS6hao8v7X1Fsp9cBs1+UnLC9iY3K96rZHuTO+t93XDrnfzuDw91vr7u5tu5+w9/TPodYwyvLX2LM6afDMC5XzmdxUteynosdmKrVBURLyEl8bAxppvKjVYcxpjFInKXiAw0xmzP5zhLHSuL9TT7ScklTyzqyyVXDsPrNfh8wh03NjDztNjnz9Q5/+wvM+t/f8DF35xFMBjkqb8+xzN/XMBxp54Vd/t7bv0ltTUD6NenT+cCREMGD2bz1m0Zj8EJ2KYoJNQecQGwxhjz6wTbDAY+NcYYETmakAXUlMdhKlhfrKd9n5RcsL3JzSVXDqO1zUVrqJEq3//pML44+T8MrMlslbwRw4YyoH8/3lu9hm3bmzjskIMZMXQI/3j2iYTfadrRnNGxnIydFsUxwCzg3yKyMvzeVcAIAGPM3cBZwMUi4gdagXNMsTWnKhCsLtbTzCfFajZsKsPrNZ1KAsDjMWzYVMbAmsxbbc/66kweefIZtjVu5+tnncnuz/Yw45wL4m57z62/5L8OPIBdu3d3Lmu6eetWhgzeL+PjOwE7s55eh5gm6923uRO4Mz8jUnrCqmI9zXxScsGIoR34fF1Fit8vjBjakdV+T/3SFP7vN7/F5/dxz29+idvtTmpRABw38SieeW4JM0+bzqNPLWL61BOzGoPd2B7MVkoPzXxScsHAmgB33NhAZUWQPr0DVFYEuePGhozdThHKyrwcO/EozpxxCm53agHya378A+6670G+cOIMmnfuZNbZX8lqDHajKUJK3smm75NVRX9KcTLztBa+OPk/bNhUxoihHVkrCYBgMMg7K9/j/jtuSfk7dSOG8+LTj/a8YYGgvzQl72Sa+WRV0Z9S3AysCWQVk4hm7YfrOefb3+XUL01h9KiRluyzEFFFodhCuplPdq/Qp5QmYw4azcpXnrd7GLajMQrFVtZWtPLEgCbWViR/ArSy6E9RlPTQRzHFNtLJfrJ7hT5FKWXUolBsI53sp0jRn98t+Lwu/G7J6wp9ilLK6K9MsY10s5/sXKFPUUoZtSgcTlmbn/5NrRl1anU6keynr2+vTbnorqPCw86aSlUSiqOZ/+AjfOHEGVSP/nxRtPTQX5uDKYV0UO37pBQjE8cfwbSTvsip533L7qFYgloUDsXKNSCcTqqZT4qSCtK0A/d7q5CmHVnvK5M24wCHHXowI4YNzfr4TkEtCofSmQ4a2JfpE0kHLSa3i/Z9UqzEu2gxVVfOBa8HfH723jgP32kzMt5fJm3Gxxw0OuPjOZXikThFRqmkg3bJfCKU+aSKQskEadpB1ZVzkbY2CHeQrfrpXFomT8TUDMhon5m0GS9GVFE4FKvXgHAq2fR9UpRoXJs2hyyJqDbjeDy4Nm0mkKGigPTbjKtFoeSVUkgH1RXvFKsIDh0CvpgYnt8fej8LMmkzXmxoMNvhlEI66Ji2Ss7aUaNKQskKUzOAvTfOw1RUYHr3xlRUhF5nYU1AZm3Gf3//wxx6zBQ2b/2UY/97JpdcOTerMdiNnUuhDgceBAYBBphvjLktZhsBbgNmAHuBbxhjVuR7rKVKvlt666p3Srb4TptBy+SJuDZtJjh0SNZKAjJrM37RN77ORd/4etbHdgp2Pqb6gR8ZY1aISB9guYgsMca8H7XNdOCg8L8JwO/C/ys5Jpc1HPEUkGY/KVZhagZkFZOIRtuMh7BzKdQtwJbw37tFZA0wFIhWFGcAD4bXyX5TRPqLyP7h7yo5IpctvRMpIM1+UpyIthkP4YgYhYjUAUcAy2I+GgpsjHrdEH4v9vtzROQdEXmnpQgL0uKRy9YeuWrpnayIMJL95DJo9pOyDxMk9JyoWEUm59P2CKmI9AaeBC4zxrRksg9jzHxgPsCBNZVFf1flurVHrmo4khURjqnQ7CelOx2NG2mpqaFvhZdQyFLJBmMMTU1NVFRUpPU9WxWFiHgJKYmHjTFPxdlkEzA86vWw8HslSz5WestVDUdPCkj7PimxNC3+PXAR22uHgzjCAeJYqtpTy8iqqKhg2LBhae3bzqwnARYAa4wxv06w2SLgeyLyGKEg9q5Sj0/kq7VHLmo4UlVAmv2kRAi2ttD45E12D6MgOP2RNTnbt50WxTHALODfIrIy/N5VwAgAY8zdwGJCqbHrCKXHfjP/w3QW+Wzt0VHhsTwtticFpNlPiuI87Mx6eh1I6nQMZzt9Nz8jKgyKobVHMgWk2U+K4jwKR7oonRRzaw/t/aQozqN4JEyJkQu3kBPQ3k+K4jyKT9IoBY9mPymKs9B8M8WR6Kp3iuIc1KJQHIdmPimKs1CLQnEcXTKfJJT5lA25bHeiKKWAWhSK47Ay8ynX7U4UpRRQRaE4Dqsyn/LR7kRRSgH9tSiOxIrMp3y1O1GUYkdjFIqjySb7KZ/tThSlmNHHKsWxZJv9VAztThTFCegvRnEsVvR9KuZ2J4qSL/RXo1hKvPWwM8Wq7KdibXeiKPlCfz2KZVidinrYTi+37hrE27UdHNzRR4vuFMUmVFEUCFY+qecCq1NRo5XOnKBh5YRyNo9URaEodmD3Uqj3AacC24wxY+N8fgLwDPBx+K2njDHX5m2ADqEQisasTEWNVTpLh8FLni1UuVyMDvaxeuiKovSA3Y+m9wN3Ag8m2eYfxphT8zMc51EoRWNWpqJGK52lw2DKBdDhBo/ZzLUN2vdJUfKNrXUUxpjXgB12jiET8tk7qFNoRhF5UncSkVRUv1vweV343ZJxKmq00nmlLqQkAi7wubLv+6QoSvo455E0MZNE5F/AZuByY8zq2A1EZA4wB6C2KrdTyrcbqJCKxqxKRY2ufzi2AcoChg7Aja54pyh24HRFsQIYaYz5TERmAAuBg2I3MsbMB+YDHFhTaWI/two73EBWFo3lIyBuVSpqtNK5vt7Pu/07dMU7RbEJRysKY0xL1N+LReQuERlojNlux3js6h1kxZO63QHxTJRUROmMDsLognNQKkrx4GhFISKDgU+NMUZEjiYUU2myazx2uoGyeVLvtaudI5ZtxR20JyBulZJaW9Ha2VH2sJ1eR6cLK0oxYXd67KPACcBAEWkA5gJeAGPM3cBZwMUi4gdagXOMMTlzLfWEnb2DMnUbDanfxbhlW3AFu76fry6qVrnrovs+eYOw5EWYuMXl2HRhRSkmbFUUxphze/j8TkLps47Bjt5BmT6RdwrpYPfP8mUJWeWu69r3CV4fAcdtCE3MienCilJMaJvxDOio8LCzpjJvlkTkidzrC+IJGMYt25pSam7c1Fog4Mo8dTVdrHLXRfo+uYJQFoQT6vd95sR0YUUpJlRROJxs6ijiCemAC16ZVpc3V41V9RWRVe/+59MB/O0hmNSw7zOnpgsrSrGgtrrDyeaJPFFMZU+/8lwNNy5Wuesiq95VDivHv0nXmFCUfKG/LoeTbQDdKesxWFVfsbailSeO8HPEqCGMb/Ro1pOi5AH9hRUA2Qr7dIS0k7vURmc+PV4jXFc2gjFtzhqjFTj5Giilid6FBUIiYW+lULG7KK8nrFjxzuk4/RoopYkqigLGSqFSCF1qrVrxzqkUwjVwCmp15Rc9wwWK1ULFrvYk6RDJfIpUZxebNVEI18AJqNWVf/TuK1CsFiqF0qU2kvlUjBTKNbCTUra67LSiivvMFjFWCxU725NkQnTfp2JRHIV2DeygVK0uu62o4j2zRU4uhIpTUml7Ijr7yWOE6zYWz6p3hXIN7KIUrS4nWFF6FxYwuRAqVtU79EQ6ZnTstsWe/ZSva1CIlKLV5QQrqnjPbolQiEIlHTM63rZjK4o7+0lJTqlZXU6woor7DCuOIx0zOvG2o7mO4s1+UnqmEB+QMsUJVlRpnGnFMaRjRifbdkxF8WY/KUosdltRqiiUvJKOGZ3KtsWY/ZRLtFCtcLHTirK1zbiI3Cci20RkVYLPRURuF5F1IvKeiHwh32NUrCWdtuM9bRvJfnp4YCM/G76BtRWt+Z5OQTGkfhdTF61n0ssbmbpoPUM+2WX3kJQCwe5HivsJrWD3YILPpwMHhf9NAH4X/l8pYNIxo5NtW+zZT7FkYw04IcVSKVzsXgr1NRGpS7LJGcCD4XWy3xSR/iKyvzFmS35GqOSKdMzoRNsWe++naLItuHJCiqVSuDj9DhkKbIx63RB+r4uiEJE5wByA2ipnTUl9wrmj2Hs/RcjGGojcfz6Py/YUS6VwKQrJZYyZD8wHOLCm0vSwed6wu+y+FCjm3k8RMrUGYu+/Tw7oy8iPWkqmUE2xDqffJZuA4VGvh4XfczzqE84fVmQ+Odnyy6TgKt79N/KjFl49pQ6vP+jIeSrOxel3yiLgeyLyGKEg9q5CiU+oTzg/wteKvk9Ot/wyKbhKdP95/UF21uw7P05WkIpzsPXOEJFHgROAgSLSAMwFvADGmLuBxcAMYB2wF/imPSNNHyeU3dtJvoRvtplPhWL5pVtwlcr9l49rpIqoOEh65USkL1BrjFkf8/5hxpj3sj24MebcHj43wHezPY4dOKHs3i7yKXyzzXwqJMsv3UyxZPdfPq6R0y01JXUS3hEi8lXgN8A2EfEC3zDGvB3++H5Ai996wO6ye7vIp/DNNvOpmC2/ZPdfrq9RoVhqSmoku2JXAeONMVtE5GjgIRG50hjzNCD5GV7hU0rNyyLkW/hmk/mUT8vPDjdMovsvm2uUyjwKyVJTeibZFXNHAsfGmLdE5ETgWREZDjgmBTUWjz9IWZtfb0Ybscvtlmn2Uz4sP6e5YTK9RqnOo5gttVIk2V2xW0RGR+ITYcviBGAhcGjuh5YZvXZ3MHXRett/iKVOvt1u2WY/5dLyc6obJt1rlM48SjlGV4wku2rfIcbFZIzZLSLTgCtzOqosEAOegHHED7HUyafbzcl9n6x2w1jpwkrnGqU7j1KN0RUjya7cQuBuEbnFGBMAEJFBwC3AGODa3A8vc9QfWlqM3VuF1wg+Y/DirL5PVrph7HRhZTKPUozRFSPJ2oyPBw4AVorISSJyKfAWsBQ4Oh+Dywb1h5YWJ63tYMn9hnmvCkvuN5z0nw67h9RJOq3VkxHt+vH6gp2Wc1mb37KxlrX56d/UGnefVs1DKTwSXmFjTDPwnbCCeAHYDEw0xjTka3CZYAT8Lr2Bo9nVVs22PUPZr9cm+lU02z0cy9knQOG4DaEnXv8mZ7kerXDD5DqTKBVrRd1JpUmyOor+wC8Jtc6YRqhC+jkRudQY81J+hpc+e/qU8cKUEXoDh3m1fgZ3vvULPC4//qCH70+4muNHPmf5ceyswHVSKmay85CtGyaXmUTpBqr191VaJLvaK4C7gO8aY/zA30VkHHCXiHzSU1W1Xfg9Lr2Jw+xqq+bOt35BR6CSjkDovTuWXc/hg9601LLIhd88HcUTLUCXDoNX6uDYDcG8ux5zHT/IZSaRk5St4jyS3QHHx7qZjDErgcki8u2cjkqxhG17huJx+TuVBIDH5WfbnqGWKYpcpH4mErjRaytEd0CNCNDWhi2cMgs63OAxcG2DjzFt+RFy+UqBzZXrR+selGQki1EkjEUYY+7JzXAUK9mv1yb8wa6X2B/0sF8v6zq1W/UkGq0E4glcT0eQse9uA8AdMARcgEinEtk8sh9Pf76dds8OggLGkNcU2Xw+kVvt+omc+1VfqGXsikate1C6oXdBEdOvopnvT7iaO5Zd3yVGYaXbqacn0VRcSNEWhCtgMDGF/0bg8yu24Y46jicI0LVe5uCOPnhMsy1LoxbqE3ms9bbqiP1oGVChgWqlC3onFDnHj3yOwwe9mbOsp2R+81R89vFcNrH9YSRoCLoEd7D78aOf2u1cGjXb+IEdyQDxzv3Yd7fxwumjVUkoXdC7wQbyna7ar6I5p8eJ5zdP1Wcfz2UTcAsEDbhCPqRV4/fj8ysa4x7bFfPUbufSqJnGD+wqotMAtpIqejfkGSvSVZ24GEys3zxVIZTIZYMIBhAR/F43HxwygDH/burSU8YAHxwyIO45sGJ51AjpnO904wd29oHa28uLqwDdZUr+SVaZnXNEZJqI/EdE1onIT+N8/g0RaRSRleF/F9oxTquITlfd6+tDR6CSO5Zdz6626pT3MaR+F1MXrWfSyxuZumg9Qz7ZlcMRZ06qPvtu1b4uQMAdNHgCBne4+njz8L6hAHYUARd8cmD3cxdpEPjwwEZ+NnwDaytaM55Hrs93p0KNIqJQk5GsgjpVBm7dA8FQRMgQOp8awFbiYdsdISJu4LfAyUAD8LaILDLGvB+z6ePGmO/lfYA5INt0Vad2IY1HOj77aJeNpyPAUf/cjDu4LyARWet55cT9U9qfVQ0C83G+MwmCW+GqiszNHXVoA2wf1Cut/aRyHKdZv0r62HnljgbWGWM+AhCRx4AzgFhFUTRkm65aaD7ldHz2EZdNWZs/oeDcWVOZ0v6yXR41Qj7Od7pBcKuUV9y5uV2Wzs1pa3AomWOndBkKbIx63UCoXUgsM0XkeOAD4AfGmI2xG4jIHGAOQG1VdlPK5RNQtumqhZiCma7PvifBmcr+rMp+ytf5TkehWqW8cj03O6xftV5yh9PP5l+AR40x7SJyEfAAcFLsRsaY+cB8gANrKjNefS8fT0DZpKuWymIwqQrOZILBiuynfJ7vVBWqVQI+13Oz0hpLtxZHrRfrsVPCbAKGR70eFn6vE2NMU9TLe4Ff5Wow+XwCyiZdNVqIRlpZFOPSrz0JzlQEgxWZT07rlmqlgM/l3KxSaJnW4jg1dleo2HkW3wYOEpFRhBTEOcB50RuIyP6RdbuB04E1uRqME/3/iZ6kOio8DNy6p2SfoFIRDNkujRqNFS0zrHSLWCngc9UJ1gqFlk0tjt2/3WLDtrNojPGLyPeAvwFu4D5jzGoRuRZ4xxizCLhERE4H/MAO4Bu5Go/T/P/JnqRS+QEV8xoUPQmGsjY/6yp2OWZp1Fy4RQqh1Xe2Ci3bWhwnx+4KDVvvNGPMYmBxzHs/j/r7SvK0PndHhYdHR01n5rqX8eHFi4+nDjiBvhUf5+PwXehJEfT0A8rXGhR2kUwwRIRyr+Hwx6+HOsm6bVwatdTdItkotHRrcYo9dmcneibD7Gqr5sKPn+CHfEYd9dRTR8tHvbl37JS8P5H3pAiS/YDytQZFKuQqCyWRYAA6hfJx9fDiA/DSAVA1fAijg/ZYE+oWyZxMa3GcEEsqNvRshokUw20P1LKdWgCqXLstXbshVXp6kkr2A9rWlPs1KFIh11ko8QRD/6bWLkJ5UgMc+amLpRUedtZYdui0ULdIdmRSi6NYj57VMPlYuyFVUnmSSvQDcsI88uVuiRUMiYTy8lo/7/ZvsryjbCoWU6G6RZxUk6AKwH707IfJx9oN6ZDKk1S8H5AT5mGXuyWeUH7wSwP46ejNlmQ/RZOOxVRobhGtSVBicfYdm2dyvXZDumT6JGX3POx0t8QK5VeGZJf9FO/JOhOLqRCeisva/PRrbuOIt7biLtHguxIfvfIx5HrthnwRPY98p8ra7W6JFsrZ9H1K9GRdjAHqyFwBXIGuSr7Q56Zkj175IseuVNl8uFtS8aNn2vcpmdVQbAHqLnONQyHPTbEGVRR5IJeBwWTWgt2psrHulsh5iLQeyeZ8pONHz6TvUzKrYWdNZUEGqBMRd66EVxpE16hQVFHknFwGBnuyFrJd/8JKol0b7oAJLUIkktVaCunECNLt+9ST1eDUAHUmDyXx5hpwwVvHDaWlusIxc1PsQ++AHJLLNNFUrAUnpMpCfNeGJwhgrFtLIYkfPZO+T6nEWdIJUOcj3TTTh5JEc92+f++cjFMpPFRR5JBcBj1TsRackCoL8c9DhHyspZDpindWWQ35SDfN9qHEqRaS4gz0bsghuQx6pmot2J0qC/HPQ4R8rKWQTeZTtmmt+So+tOKhpBBSeBV70Lsih+QyTTQda8HulN/o8wDdYxS5Xksh08wnK9KK85VKW2yZWIqzUEWRY3Jp0mdrLVhdX5Fsf/EWXMrnWgqRzKc9Vyyh7LoTe7we0YkCvoCHrx56N6cc+Oe0z1O+BLjdtSt24aRWI8WMGJPxyqGO5MCaSvPrU+rsHkbeSVfoW11fYWdr83TmvuWQGWx/8XaO21TG5A1Bls29mQ3TzgCg100nd+7vwkUv0hGItjwMZe52vj/hqrTnNeSTXd0EeK5aYpSS4NRWI105/ZHs1nUTkeXGmCPjfmanohCRacBthBYuutcYc2PM5+XAg8B4oAn4mjGmPtk+S1FRpCuk4wlCr6udW6d9meH90l9/I97+ytyt3Ht67lu0P/fh2dy74iq8Lj8B40469/Wu3Vxdt4kON5QFQm3Ij2qsYNHiN2mv3tdedv1qLzdcXEvrZ65u+8h0XqUkwPNBWZufqYvWd8mk87uFF04fXbLnN5eKovsvIU+IiBv4LTAdOAQ4V0QOidlsNtBsjDkQuBX4ZX5HmRq72qr5sGksu9qqbTl2JE12r68PHYFK7lh2fdKxRDKmovEFy7js+ad47ZPpaY8h3v4iGVi55PkPz+bud+bhD1bQ6u/d49zXlO2hwx2qEehwwSt1EPR46bV5Y5ftaocECPjiH9NdUc4nX3maPVcsYc8VS1Iea0eFh501lSUrxKymM/YTRST2o1iPnXft0cA6Y8xHACLyGHAG8H7UNmcA14T/fgK4U0TEOMhfZvdqcpkU1cXLmALBH6zIqHLbjnqNXW3V3LviaqCrsHARSDj3gzt6URbYSYeBsiCcUA8uv489Q4Z3btPS7KJxs5tZP9rJgzf3x9chXY4R8IcUSYSIsihvbqLX5o3sGTK80zqJuLIU69HgfX6xU1EMBaIf5RqACYm2Ca+xvQuoAbbnZYQ9YHeLDMhMSEcypm5/8//wBcuIFoSZVG4ny8DKVUPCkIL04QuWd3m/LdCL9TsO5qCaVd2+MzrYhztX1LBlTxPHbaniqMYgyy+f22lRvPzmMOZfW43bCwEfzLpiJ7ubXSxc0BePJ6Qk5sxtpm91sMt+Rzy3kAnXXkHA48Xl8/HqFbfQOPP0hBaHKpDsKdXgvV0UxVkVkTnAHIDaqvxNyQktMjItqjt+5HOM6r+Wy55/Cn+wovP9TC2BeBlY6VhbEaEaeaKvHRLoJpCj6d3sIvBqFfhjPxEW/PsaDv/ld+J+vxI4uLmJ1s0bWb7m34y/eR5Bb0jAP+VfQEfgPGgPbfvHm/tz++KtTJm5N+GYypubmHDtFXja2/C0twEw+frLuc5M4bCzeiWdayyqQNLDqUWCxRiPsnMWm4DhUa+Hhd+Lt02DiHiAfoSC2l0wxswH5kMomJ2T0cbBKS0yekqTTSSYBgDfOXYP8+eV4e58Yt6LZ9qf2JPBODzAkPDfW5pd3DljMB0B1z5ra/nNHHTd5QkVwBvPVXZ5op8zt5nJ01qBrgqklkbqNm/khz86iJtvOgi/r6t7yO2Bxs3uhMdpr67h3U3/pvyfPydY62dSQ0jAz+dC/sbJnUvhikD9Wg+HTerotq/IeA5p2UTA4+1UEgA+vLx6007qpvRJquxiUQWSPk4rEizWTCw7z/DbwEEiMoqQQjgHOC9mm0XABcBS4CzgJafEJ/ZcsQQPMOeovcyfV26JoM2GaCGdzrEnT2tl7IT2lJ7i06Fxsxu3l86nc0guwFuaXcy/tpqOdlfnd+bPq2bshHZWvVneqUBmtj7KAte3odzDFJ+Pyd/9NTPv+g6+jn37io0jxPLIk6tZzIXIF/3ccGwo+2lSQ0jA11HfqSja24RbfjiQi6IUFnRVaNUd4/nvYFezxouPBm8djZux5HwmC5pnokTyvT5JqZCvKnw7sG304ZjD94C/EUqPvc8Ys1pErgXeMcYsAhYAD4nIOmAHIWWSlOCgg9hzxeJcDr0LuRK0+aRvdTCjcccL4EaIlzmUTIAnUiz1a72dCmRgeyO/59uUBVohvO/pv/shP7r8JH59y0FRyrp7HCHCi09U8exry+EkP7igw4SynyY1QKXXR4MZAX5DyEIRfO3SqbD6Vge7KbRWBvEt973cw4X48OLFx7dYwKeBWmqHbE37nKZKp4V14Ytx55pIgdidfFHMFOOCVhFsHb0xZjGwOOa9n0f93Qacne9xpUumgraQiQRwI/796MI1CJ2TOXObmT+vOiUBnkixgOlUIHXU00EZVex7ug96vJxw8IfULe6TUFlHhGpFleGBm/rDoBMhUIaYdsqCQY6sr8JXFuQP029j21/26za2aEsonkJ7uuIcKr56NGv+2ESDt45PA7VJ55otyVx0EeJZIS3NLu78UrWtyRfFTDFnYhW2mlNsITqAS9g3P2He5Xw64dgulkU61lYixVI3xt+pQOqpo4yOLt+LpLcmUtbRQtXfISAGGibBAy9i6l6hrP5wfr51IId/s5rf33Mgsem2AH7fPksootAG0kgd9dRTR4u/hqPPr+Lo86to3Ay1Q7bmREmUNzfB2k08NW88HR3dXXQ9HbNxsxt3ZTl8tu+9SF3IYYtPsHy8pUYxZ2IV/gyUvNNr80aCXm+nkoB9hWuxLqh0rK1EiiWiQPZ4apjTdi/3uS6EMg8uf8iSiT1mhHhxj04aJkHDJHZhOOCyXSz4bV/iKQkwnHlhS+dY+lYHufvMe/na45d1upoeP/M2Kqqnd36eCyIWnN/lZVqHn2+xgMc5F+g5eB8hmTtQA+nW4NRMrGwpjlkoeWXPkOG4fF0lTmzhWqbEUyxdFcgJPMvShLGRaOK5icrKDcGgweMF335LOeL85+k1ciKesun44xT1esoMU2bu7Xxd3tzE2QsvxUMbhF1gsxZewqI5XduApJrmmwpdUnAJKef7mM2LTGU7tT0G7yOk6w4E6wPppYDTMrGsoLhmo+SF9uoals29mQnzLifo8fb4ZJ8pscI2ItDaqUnpWIlacfzfY9v4YMdy7t9yFiuCPv61xYsZ9AJ8dEzUVgaX2/Cda2IE6dpN+F3eToEN3a2pVGII6RDPgvPhZUzFR7xlatKKh1iZfKFWiHMoa/PD229DXR3U1lq+f1UUDsHKJ9CeSJatlCobpp3BpxOOzXo/ibBC2CZ6gh46ys87/n8Q2OQjSACCMHLqEj6eP7nzu184fi9z5u7qci3eeK6Sp+aNZ1pH13TYaGsqWZpvptc1ngXXq7yDL9/Sm/PHdI2HpHIf5Tr5QhVIfulcj/6lk6GjAxYsgHPPtfQYqigcgNVPoMnoKVspFfYJo1raD81OQcQTbNkK200fe1i3ysuBY30Jn6APrpmEx+XFHwQxZXy8+EtExyhWLasEdnUZ5/xrq+nocPEtFnAfs/HhpVd5RxdrKp67K7pwLxMSWXD7T+oHdFVk+bqPMiFRXyxVIJnTpXZjV/h+nT0bpk611LJQRWEzuXgCTUSq2UrJsFIYJdpXusV60fzhxn4s+VPvztdf+tpnfOMnu7p973MDxnPV5MdY2fAmf5l3GjRM7vK5y931eNFjepxzeZGp1PExh503gKnT9rVAiefuSlS4l4xYBdqTBZfP+ygb4j6oqAWSMXHXo/d6ob5eFUUxkY1QTIeWZhee17d0azeRKFsp0T4SCaPIXJK5PKKFH5BwX+kW60XY9LEnrCT2WQZ/f7w3J391D0NHdWsKxecGjMe9ZSLPb6slds/RKbEQ+js62L2dWrZTy78eDnL0+fvcPxF31++vqY7qPNu9cC8ZiRRoe3Xi2Ey+7qNsSPdBRQPpPRN3PXqfLxSrsBBVFDaTqVBMh4jgGeRxc+6exP71nkgkjF58oopn7uub1MqIFX5nzG5JKNhGH+pLOzsHYN2q+IVN61Z54yoKCCuAQUthyGtQf0IobRbDBVfs7HK8vtVBzpzdwp/v6keXbrve7sJ48rRWevcLcuvlNbS3pt6DCjK3DPJxH2VLOmnVPaFKJER07YanV5+QkliwwPKAtioKm8kkZTEdogXPxvZBnf51d5UHdyC9bKV4wsjvh4UL+uJLUgAWT/g9fU9fJGbZrGjBlkl2zoFj4y9ak+h9gK3mbfifcyDog0AZ7kde4ILzxnRJiY0wZeZeFt7bN2wpdB9zNHVjfNQEGhnCBuqpYzu1VPsaOaRlOTQPzcgyiHwe73zk+j6yglymVXc5Tom5siK1G9OufESznoqZXPaLihU8j3Mub1SdxNyfvEv1sfun9SQXTxidMbuFZx/s26UpX+yTczzh5/cJE07ey7uvVSYUbOlm5wwd5edLX/uMvz/eNUaRyJoAWNO0lCA+cAUQVwenzf0LUw8bkXD+F12TmjAe++ZTfGx+TGu4KO8P8k2+bf4AP/EkTSJIZBl8vMbLdRfWJrXanN53LF9p1YkoZgXSUeGBo47K2f5tXTM7FxxwyGHm+ofz1xTQ6bQ0u7hkxuDQ03yYsvIgty/OvM1EbKyhp/23NLv4/vTBIasjCm95kBse2UbbXrFUsEVnPSVTEgAf7FjODW+cgz/ow+PyctXkx/jcgPFJvxM7/1jBXN7cxOkzJnaJBUXaDEbwl3dfqzvCG89XdlFGs360k4du6W/pNbQTK9Kz80UhKZFcrpmtFkWRkwuXROyTfuz+z798Z6erJLLtmRfG8e97oG2vMPpQa9c5HjrK36OCiBDJflrTtJSDayb1qCRg35wSBZ3j+eJjCXq8sHYT6/sO7qYkYy2DeBZZbFZWIZEsKO80itkKSQdVFCVArl0S0fuvX+PloZv7dxOeU2buDcUy2nv27+ebzw0Yn1BBJCpgSxZ0Lo/ji+9Gh5/v/2A8zWW1cV1Jsco4+rxBSMHWr/FarmTtJJ9Fp9lSagpEFUUaFJLJHEuuq3Ej+77uwtqEGTsXOTjY+sGO5d2simQ1I0nTUQ/t7otff+Y5jF74WOfrb/nvZXNgEJFmuMkym3bvdBEIxDqvhAdv7s9RU9pSPodWCmKrhbrTiwVTpVgViCqKFLGiornY6SmXPxvLJpdPm/HiFIPlqKRpqj2lo8YrkFs95zJ6bd7I6pZRPPWTg7u2+446T9FzXfVmOb+fNwATZ8ouV+rup1wXSmZjsRZKsWA2FLoCsUVRiMgA4HGgDqgHvmqM6bZyiogEgH+HX24wxpyerzFGY0VFcymQSi5/JpZNIiFnlYW3pmkp/mCo75M/GHrt7piIy0On4BpII2PkI/as7U3fSf1Siv3E+uIjr3s1uxKep67rZ4AxEl4TvDuBgFBR1XMyipWCON6+fvezalyeUE1JJkqoEIoFc0Wh1IPYZVH8FHjRGHOjiPw0/PoncbZrNcaMy+vI4mBloVAxk03gPJHQTyTkzvjsMU645XJLLLzovk8el5eDaybx8Ute2vaEBPQ5PMoCZuNr89Lrhx2dx8rUQkp0nqB7tXooXyqaqGU2MVx17n5cdE1ywZxpbUaq+woEhEBA8GWohAqhWNAOnGSF2KUozgBOCP/9APAK8RWFI8hXoVCuSCaErXTnlDc3cerwjRz5yEg27t0v5f0mc+vFE0z7uRo5/ldX4PFbY+HFZj4NlqO44Zb+gDCQRhYwO7z8aiu0dz1WprGfeEpm/Wpvt7kmRgj4hAChJ/pkgjmRIK5PoTYjlooq06kQEpGuNVAIxYJOIpkVkivsUhSDjDFbwn9vBQYl2K5CRN4B/MCNxpiF8TYSkTnAHICBg4daPFT7C4WyIZEQtjp4GPc4h+57wk+klHpy68UTcoNbP6EVL2Uxa2dnY+FFZz6tX71POSVap9sKazJWycSbq9ttQCLrh0d6R3UlEJCk3WnjCeLzL9/JQzf3T8sdFblnXC4Ag9ttCATijCcDa8DpxYKlTs4UhYi8AAyO89HV0S+MMUZEEjlaRxpjNonIAcBLIvJvY8z62I2MMfOB+RAquMty6HHJ9foLuSCREF73X8cz/9ohlgUPexL2yZRST269aCHncofTQhmFl9xYeB/sWM7KwJv49jsVPjsm6TrdVhNPoM/60U4euKma+Mu0RpP881RqM5JZAl1cgGECgdjjGrzlJmNrwIrMvEJKsS0kcqYojDFTE30mIp+KyP7GmC0isj+wLcE+NoX//0hEXgGOALopinxRSIVCkFgI+1dtwu091LLgYTJh30ht0kBqKm69iJB79/VyHvhVNdv31HZZE6KqrIO3LLDwItlPvqAP16zbcD+0hD2fTkprne5siSfQPWUmYUAbQkHkujE911PECuJ04gLxFEss5RWGH9yyPeN1N7KlWFJsnYir501ywiLggvDfFwDPxG4gItUiUh7+eyBwDPB+3kZYBCQSwp6xQ1MSEi3NLtav9tLSnPw2SSbsOwVMFNGB1Ihbz19eQUevPvjLK+IK4r7VQY44tp1guOD6cc5lJJ8w3ft3Hnn0LUtSldc0LcUX9GEIEAh2YIa/yqkXtHDc30/g2eeX8vLdj7Bo8Zs5T4vuWx1k9KG+hGm4YHB7DOWVQbzlQS6/4gPqNq+gvLkprWPMmdtMWXmQyl5BysqDSS2BRMvKdhmVgboxqVXEW020xdP6mYuOdhfz51X3eO8qqWFXjOJG4E8iMhv4BPgqgIgcCXzHGHMhcDDwexEJElJoNxpjVFGkQaLYStmo6h6Dh+k8ncU7zqLZt9JIbUoZLam69WJdMy3+GsbPdVE2ypqnxpFlkzG+MnB1QLCM4PoTeebNvkyZuZdGanmfwdQSoC/5c2mserMcY4RItpPLDd/4aTNHn9RG42Y3E9c8kXH2VzpxgXhusRPO3MMrC3s5IgBdyim2+UCbApYA6WY9ZdpIsOPjZtY+0sgDiw5lZ/m+1hRAN6WUiUsgMo+tVellVaXK+tVerrt2HR2D9q1NUdkryKkXtPDMguTrbYD1/vF418FbHuSO8HWI13wwWbNBK4htuOiUmEAuml8WGuceMSyr72tTwBInUWwlUfAwk6ezN56r5PfXDgn3JBIiseb586q5ffFWbl+8NSuB0lNWlRXUDgkQ+GgSfHhM53vtreH1NnoI/OfCPx7vOniirkOm9T2ZCvdEc7RaEGcyPk2xzS2qKJRupFsAFfEP+9q7+4OjV62L/dGmKhDyWRkvLoH9l0LdK1B/ArJlIm5P19qBWKWZauVzupXkPV2HTOp7MlVo+WqzkY3CjbjS6td6AWNbvKQY0UiP0o10A53xAtYREimYN56r5JIZg7nh4loumTGYN56vTDiezifnKCJPzlbSuNmNq+4NuGAKnPQzuGAK7lFvEIjJOIqdU08BewhZRKfPmMiJF5/H6TMmMuL5bvkb3ejpOqSaCBAhm4BvKnPMFisC0qveLOfXP6zhtp8M7PG+UlJHLQolLukEOhNl5njL4ufUp/t0mq/K+NohAQJDXwV3B7gCYDoIDHuVC752MH+8uX9Cl0ZPT/7ZWEQ9XYd06nuyCfjmo81GtgHpUmguaBdqUSgJiU7T7Gm76Cdfb3mQs/93F3c8tzWu2yDdp9N0n5wzpW91kDOnfgECZRBwQ7CMM6d+gakz93L74q1cdXcjty/uPqeenvyzsYjKm5uo27yCQ4YkDsq2V9ew49BxPZ6PbIR9ulZmJmSrjPJh9ZQqalEolpCtBdKTQMhXZfzMGWMZ/cljrNi4jC8Mn8C4kWOBnquGk80/U4vI6tb26QZ8Y2NIuW6zkW1AWpsL5g5Njy1ynLrYUuy60MVeRTvi+We61bMkE/q5TH1NJYnAzirnbFJuS+2+ikbTY5WMcPJiS4XQBC7eqneZkq5FlMvW9j1ZR3b7+rPp+VQI91UhooqiSCmExZasaAKXK+KtepetskinV5idre0LvcrZyfdVoaLB7CIlXymlTiPV/lQ90XXVOx9rmpZaNMLUyFcAPx7q61diUYuiSCn0xZYywUq/erxV7/KNXa3ttcpZiUUVRZFSyIstZYLVfvXBchQX7P8Ejd7XGTdsYtZup0yxq7V9Mfn6ndKPqpBRRVHEFOJiS5lipV99n2UynYBvOsPmNvO5EsmciaYYfP26RoU1qKIocgptsaVMscqvHs8yufvetTQMedZWy0JJH7uzt4oJDWYrRYFVlcPdqnuHLcV/7sk8U38Tv/jnOaz8ZIW1A1dyhlZqW4daFEpcCtGva4VfvZtlUvcKuDswBPD74ZbbVnHxl45R90UBoNlb1mGLRSEiZ4vIahEJhle1S7TdNBH5j4isE5Gf5nOMpUw6nV2dRqr9qZJ9P9oykQ3Hd+n9FFh3oi6xWSDkoz9VqWCXRbEK+Arw+0QbiIgb+C1wMtAAvC0ii3Q51Nyift3odQ083HTZJAIPvNi5PgUNk3BVBQum+KzUKabsLTuxRVEYY9YAiEiyzY4G1hljPgpv+xhwBqCKIocUelVuMpK502I/61sdpFdfg7cMAg2ToGFfHUXAL53ui8j3KqoMbXvFMcKoEF2HuaIYsrfsxskxiqFAdBlxAzAh3oYiMgeYAzBw8NDcj6yIKVa/brI0yUSf1Q4JEIwskjYssurdF5l1wRj6Vgc7v2cAX7tQVh5qsGl3CqamhCpWkzNHq4i8ICKr4vyzvCudMWa+MeZIY8yRfaoHWL37kqIY/brJVk5L9lnkXHgO+Gfnqnfu2VMZceI/unwvtASs0NGe2aps+ZqromRKziwKY8zULHexCYjuNzEs/J6SY4rNr5vMnQYkdbVNntZKw5BneaY+lPlk6GBN01LcHRO7fS/e9/NNMbsOFftwsuvpbeAgERlFSEGcA5xn75BKh2Ly6/bkTuvJ1TZu2EQWb+ja96lW4i3/Gv/7+aRYXYeKvdiVHvtlEWkAJgF/FZG/hd8fIiKLAYwxfuB7wN+ANcCfjDGr7RivUtgkc6el4mr73IDxXDX5Mc4++PLOduPR3/OWBQFDWbn9rrpidB0q9qMr3CklQzpZT+nuU7OeFLvRFe4UxQKSudNSdbXFrnrnVBedU8elFCaqKBQlRXKx6p2iFAKaM6coKWL3qneKYheqKBQlRSKr3rlw27bqnaLYgbqeFCVFItlP0TEKRSkFVFEoShp8bsB4VRBKyaGuJ0VJkw92LOeZD+/kgx3L7R6KouQFtSgUJQ0080kpRdSiUJQ00MwnpRRRRaEoaaCZT0opoq6nEqC8uYlemzeyZ8hw2qtr7B5OQaOZT0opooqiyBnx3EImXHsFQa8Xl8/Hsrk3s2Ga5UuClBSa+aSUGup6KmLKm5uYcO0VeNrbKPtsN572NibMu5zy5ia7h6YoSgFRdN1jRaQR+MTuccQwENie74P2hqqD4HMucEfeC0LgQ/jgM9ibwS5tmUeOKJa5FMs8oHjmUqjzGGmMqY33QdEpCiciIu8kat9bSBTLPKB45lIs84DimUuxzCMadT0piqIoSVFFoSiKoiRFFUV+mG/3ACyiWOYBxTOXYpkHFM9cimUenWiMQlEURUmKWhSKoihKUlRRKIqiKElRRZEnROQmEVkrIu+JyNMi0t/uMWWCiJwtIqtFJCgiBZcCKCLTROQ/IrJORH5q93gyRUTuE5FtIrLK7rFkg4gMF5GXReT98H11qd1jyhQRqRCRt0TkX+G5zLN7TFahiiJ/LAHGGmMOAz4ArrR5PJmyCvgK8JrdA0kXEXEDvwWmA4cA54rIIfaOKmPuB6bZPQgL8AM/MsYcAkwEvlvA16QdOMkYczgwDpgmIhPtHZI1qKLIE8aYvxtj/OGXbwLD7BxPphhj1hhj/mP3ODLkaGCdMeYjY0wH8BhQkI2vjDGvATvsHke2GGO2GGNWhP/eDawBhto7qswwIT4Lv/SG/xVFtpAqCnv4FvCc3YMoQYYCG6NeN1CgQqkYEZE64Ahgmc1DyRgRcYvISmAbsMQYU7BziUa7x1qIiLwADI7z0dXGmGfC21xNyNx+OJ9jS4dU5qEoViIivYEngcuMMS12jydTjDEBYFw4Bvm0iIw1xhR0HAlUUViKMWZqss9F5BvAqcAU4+AClp7mUcBsAoZHvR4Wfk+xERHxElISDxtjnrJ7PFZgjNkpIi8TiiMVvKJQ11OeEJFpwI+B040xmXRuVbLnbeAgERklImXAOcAim8dU0oiIAAuANcaYX9s9nmwQkdpINqOIVAInA2ttHZRFqKLIH3cCfYAlIrJSRO62e0CZICJfFpEGYBLwVxH5m91jSpVwMsH3gL8RCpr+yRiz2t5RZYaIPAosBf5LRBpEZLbdY8qQY4BZwEnh38VKEZlh96AyZH/gZRF5j9BDyRJjzLM2j8kStIWHoiiKkhS1KBRFUZSkqKJQFEVRkqKKQlEURUmKKgpFURQlKaooFEVRlKSoolCUHBDuivqxiAwIv64Ov64TkedFZKeIFEXqpFL8qKJQlBxgjNkI/A64MfzWjcB8Y0w9cBOh2gFFKQhUUShK7rgVmCgilwHHAjcDGGNeBHbbOC5FSQvt9aQoOcIY4xORK4DngS8ZY3x2j0lRMkEtCkXJLdOBLcBYuweiKJmiikJRcoSIjCPUGG4i8AMR2d/eESlKZqiiUJQcEO6K+jtC6ytsIBTAvtneUSlKZqiiUJTc8G1ggzFmSfj1XcDBIvJFEfkH8GdgSrjz6ym2jVJRUkC7xyqKoihJUYtCURRFSYoqCkVRFCUpqigURVGUpKiiUBRFUZKiikJRFEVJiioKRVEUJSmqKBRFUZSk/H/3OPeRAWnyTwAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -542,6 +542,10 @@ "outputs": [], "source": [ "from mlprodict.npy import onnxsklearn_class\n", + "from mlprodict.npy.onnx_variable import MultiOnnxVar\n", + "import mlprodict.npy.numpy_onnx_impl as nxnp\n", + "import mlprodict.npy.numpy_onnx_impl_skl as nxnpskl\n", + "\n", "\n", "@onnxsklearn_class('onnx_graph')\n", "class TwoLogisticRegressionOnnx(ClassifierMixin, BaseEstimator):\n", @@ -580,20 +584,175 @@ " def onnx_graph(self, X):\n", " h = self.hyperplan_.astype(X.dtype)\n", " c = self.centers_.astype(X.dtype)\n", - " \n", - " sign = ((X - c[0]) @ h >= 0).astype(x.dtype).reshape((-1, 1))\n", "\n", - " prob0 = self.lr0_.predict_proba(X)\n", - " prob1 = self.lr1_.predict_proba(X)\n", - " prob = prob1 * sign - prob0 * (sign - numpy.array([1], dtype=X.dtype))\n", - " label = nxpy.argmax(prob, axis=1)\n", - " return label, prob\n" + " sign = ((X - c[0]) @ h) >= numpy.array([0], dtype=X.dtype)\n", + " cast = sign.astype(X.dtype).reshape((-1, 1))\n", + "\n", + " prob0 = nxnpskl.logistic_regression( # pylint: disable=E1136\n", + " X, model=self.lr0_)[1]\n", + " prob1 = nxnpskl.logistic_regression( # pylint: disable=E1136\n", + " X, model=self.lr1_)[1]\n", + " prob = prob1 * cast - prob0 * (cast - numpy.array([1], dtype=X.dtype))\n", + " label = nxnp.argmax(prob, axis=1)\n", + " return MultiOnnxVar(label, prob)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TwoLogisticRegressionOnnx()" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = TwoLogisticRegressionOnnx()\n", + "model.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1,\n", + " 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0,\n", + " 0, 0, 0, 1, 0, 0], dtype=int64)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.predict(X_test.astype(numpy.float32))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.69516504, 0.30483496],\n", + " [0.46722147, 0.53277856],\n", + " [0.13954118, 0.86045885],\n", + " [0.7356125 , 0.2643875 ],\n", + " [0.7948843 , 0.20511569]], dtype=float32)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.predict_proba(X_test.astype(numpy.float32))[:5]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It works with double too." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.69516502, 0.30483498],\n", + " [0.46722146, 0.53277854],\n", + " [0.13954117, 0.86045883],\n", + " [0.73561251, 0.26438749],\n", + " [0.79488431, 0.20511569]])" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.predict_proba(X_test.astype(numpy.float64))[:5]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And now the conversion to ONNX." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "onx = to_onnx(model, X_test[:1].astype(numpy.float32),\n", + " options={id(model): {'zipmap': False}})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's check the output." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'label': array([0, 1, 1, 0, 0], dtype=int64),\n", + " 'probabilities': array([[0.69516504, 0.30483496],\n", + " [0.46722147, 0.53277856],\n", + " [0.13954118, 0.86045885],\n", + " [0.7356125 , 0.2643875 ],\n", + " [0.7948843 , 0.20511569]], dtype=float32)}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from mlprodict.onnxrt import OnnxInference\n", + "\n", + "oinf = OnnxInference(onx)\n", + "oinf.run({'X': X_test[:5].astype(numpy.float32)})" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, "outputs": [], "source": [] } diff --git a/_doc/notebooks/numpy_api_onnx_ftr.ipynb b/_doc/notebooks/numpy_api_onnx_ftr.ipynb index d8d65dfeb..33c97adb2 100644 --- a/_doc/notebooks/numpy_api_onnx_ftr.ipynb +++ b/_doc/notebooks/numpy_api_onnx_ftr.ipynb @@ -237,7 +237,7 @@ "from mlprodict.onnx_conv import to_onnx\n", "try:\n", " onx = to_onnx(pipe, X_train.astype(numpy.float64))\n", - "except RuntimeError as e:\n", + "except (RuntimeError, TypeError) as e:\n", " print(e)" ] }, @@ -259,7 +259,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -296,16 +296,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -333,7 +333,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "4.2 \u00b5s \u00b1 541 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "4.61 \u00b5s \u00b1 821 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -350,7 +350,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "16.9 \u00b5s \u00b1 2.04 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "15.5 \u00b5s \u00b1 723 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -376,7 +376,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -435,7 +435,7 @@ "data": { "text/plain": [ "Pipeline(steps=[('functiontransformer',\n", - " FunctionTransformer(func=)),\n", + " FunctionTransformer(func=)),\n", " ('standardscaler', StandardScaler()),\n", " ('logisticregression', LogisticRegression())])" ] @@ -469,16 +469,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -507,7 +507,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "6.1 \u00b5s \u00b1 126 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "6.15 \u00b5s \u00b1 116 ns per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" ] } ], @@ -527,7 +527,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "22.4 \u00b5s \u00b1 4.14 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100000 loops each)\n" + "22.1 \u00b5s \u00b1 2.21 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" ] } ], @@ -551,7 +551,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "508 \u00b5s \u00b1 109 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "433 \u00b5s \u00b1 53.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -569,7 +569,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "568 \u00b5s \u00b1 83.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "357 \u00b5s \u00b1 13.6 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -601,9 +601,9 @@ { "data": { "text/plain": [ - "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", - " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", - " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" + "array([[2.2948687 , 1.0178018 , 0.14858188, 1.0178018 ],\n", + " [2.4203196 , 2.2417266 , 1.7303984 , 2.2417266 ],\n", + " [3.0329118 , 1.2578778 , 0.75200284, 1.2578778 ]], dtype=float32)" ] }, "execution_count": 20, @@ -656,9 +656,9 @@ { "data": { "text/plain": [ - "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", - " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", - " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" + "array([[2.2948687 , 1.0178018 , 0.14858188, 1.0178018 ],\n", + " [2.4203196 , 2.2417266 , 1.7303984 , 2.2417266 ],\n", + " [3.0329118 , 1.257878 , 0.75200284, 1.2578778 ]], dtype=float32)" ] }, "execution_count": 21, @@ -713,16 +713,16 @@ { "data": { "text/html": [ - "
\n", + "
\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 22, @@ -794,11 +794,11 @@ "Onnx-Concat(Co_output0, Si_output0) -> Co_concat_result0\n", "+kr='Co_concat_result0': (2, 4, 4) (dtype=float32 min=-1.0 max=1.0)\n", "Onnx-Transpose(x) -> Tr_transposed0\n", - "+kr='Tr_transposed0': (4, 3) (dtype=float32 min=-2.9092655181884766 max=0.9116002321243286)\n", + "+kr='Tr_transposed0': (4, 3) (dtype=float32 min=-2.0982813835144043 max=1.0294874906539917)\n", "Onnx-MatMul(Co_concat_result0, Tr_transposed0) -> Ma_Y0\n", - "+kr='Ma_Y0': (2, 4, 3) (dtype=float32 min=-4.017709255218506 max=3.2469072341918945)\n", + "+kr='Ma_Y0': (2, 4, 3) (dtype=float32 min=-3.032911777496338 max=2.2948687076568604)\n", "Onnx-Pow(Ma_Y0, Po_Powcst) -> Po_Z0\n", - "+kr='Po_Z0': (2, 4, 3) (dtype=float32 min=0.0 max=16.141986846923828)\n", + "+kr='Po_Z0': (2, 4, 3) (dtype=float32 min=0.0 max=9.198554039001465)\n", "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst3\n", "+kr='Sl_Slicecst3': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst4\n", @@ -806,11 +806,11 @@ "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst5\n", "+kr='Sl_Slicecst5': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Slice(Po_Z0, Sl_Slicecst3, Sl_Slicecst4, Sl_Slicecst5) -> Sl_output0\n", - "+kr='Sl_output0': (1, 4, 3) (dtype=float32 min=0.045248985290527344 max=16.141986846923828)\n", + "+kr='Sl_output0': (1, 4, 3) (dtype=float32 min=0.020312773063778877 max=9.198554039001465)\n", "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst1\n", "+kr='Sq_Squeezecst1': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Squeeze(Sl_output0, Sq_Squeezecst1) -> Sq_squeezed0\n", - "+kr='Sq_squeezed0': (4, 3) (dtype=float32 min=0.045248985290527344 max=16.141986846923828)\n", + "+kr='Sq_squeezed0': (4, 3) (dtype=float32 min=0.020312773063778877 max=9.198554039001465)\n", "Onnx-Identity(Sl_Slicecst) -> Sl_Slicecst6\n", "+kr='Sl_Slicecst6': (1,) (dtype=int64 min=1 max=1)\n", "Onnx-Identity(Sl_Slicecst1) -> Sl_Slicecst7\n", @@ -818,25 +818,25 @@ "Onnx-Identity(Sl_Slicecst2) -> Sl_Slicecst8\n", "+kr='Sl_Slicecst8': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Slice(Po_Z0, Sl_Slicecst6, Sl_Slicecst7, Sl_Slicecst8) -> Sl_output02\n", - "+kr='Sl_output02': (1, 4, 3) (dtype=float32 min=0.0 max=10.542407035827637)\n", + "+kr='Sl_output02': (1, 4, 3) (dtype=float32 min=0.0 max=4.499505996704102)\n", "Onnx-Identity(Sl_Slicecst2) -> Sq_Squeezecst2\n", "+kr='Sq_Squeezecst2': (1,) (dtype=int64 min=0 max=0)\n", "Onnx-Squeeze(Sl_output02, Sq_Squeezecst2) -> Sq_squeezed02\n", - "+kr='Sq_squeezed02': (4, 3) (dtype=float32 min=0.0 max=10.542407035827637)\n", + "+kr='Sq_squeezed02': (4, 3) (dtype=float32 min=0.0 max=4.499505996704102)\n", "Onnx-Add(Sq_squeezed0, Sq_squeezed02) -> Ad_C0\n", - "+kr='Ad_C0': (4, 3) (dtype=float32 min=0.045248985290527344 max=16.141986846923828)\n", + "+kr='Ad_C0': (4, 3) (dtype=float32 min=0.022076575085520744 max=9.198554039001465)\n", "Onnx-Sqrt(Ad_C0) -> Sq_Y0\n", - "+kr='Sq_Y0': (4, 3) (dtype=float32 min=0.21271808445453644 max=4.017709255218506)\n", + "+kr='Sq_Y0': (4, 3) (dtype=float32 min=0.1485818773508072 max=3.032911777496338)\n", "Onnx-Transpose(Sq_Y0) -> y\n", - "+kr='y': (3, 4) (dtype=float32 min=0.21271808445453644 max=4.017709255218506)\n" + "+kr='y': (3, 4) (dtype=float32 min=0.1485818773508072 max=3.032911777496338)\n" ] }, { "data": { "text/plain": [ - "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", - " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", - " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" + "array([[2.2948687 , 1.0178018 , 0.14858188, 1.0178018 ],\n", + " [2.4203196 , 2.2417266 , 1.7303984 , 2.2417266 ],\n", + " [3.0329118 , 1.257878 , 0.75200284, 1.2578778 ]], dtype=float32)" ] }, "execution_count": 23, @@ -857,7 +857,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "29.6 \u00b5s \u00b1 8.88 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" + "25.3 \u00b5s \u00b1 5.76 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 10000 loops each)\n" ] } ], @@ -874,7 +874,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "299 \u00b5s \u00b1 17.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "288 \u00b5s \u00b1 12.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -898,7 +898,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "1.9 ms \u00b1 45.7 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "1.72 ms \u00b1 32.3 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -916,7 +916,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "3.95 ms \u00b1 443 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)\n" + "5.32 ms \u00b1 206 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 100 loops each)\n" ] } ], @@ -942,9 +942,9 @@ { "data": { "text/plain": [ - "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", - " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", - " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" + "array([[2.2948687 , 1.0178018 , 0.14858188, 1.0178018 ],\n", + " [2.4203196 , 2.2417266 , 1.7303984 , 2.2417266 ],\n", + " [3.0329118 , 1.257878 , 0.75200284, 1.2578778 ]], dtype=float32)" ] }, "execution_count": 28, @@ -972,7 +972,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "73.1 \u00b5s \u00b1 23.5 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n" + "149 \u00b5s \u00b1 54 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1 loop each)\n" ] } ], @@ -996,7 +996,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "283 \u00b5s \u00b1 13 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" + "217 \u00b5s \u00b1 18.9 \u00b5s per loop (mean \u00b1 std. dev. of 7 runs, 1000 loops each)\n" ] } ], @@ -1065,9 +1065,9 @@ { "data": { "text/plain": [ - "array([[2.2402291 , 1.8694952 , 1.3286328 , 1.8694952 ],\n", - " [4.0177093 , 3.5420892 , 1.1255382 , 3.5420892 ],\n", - " [1.602722 , 1.3015178 , 0.21271808, 1.3015178 ]], dtype=float32)" + "array([[2.2948687 , 1.0178018 , 0.14858188, 1.0178018 ],\n", + " [2.4203196 , 2.2417266 , 1.7303984 , 2.2417266 ],\n", + " [3.0329118 , 1.257878 , 0.75200284, 1.2578778 ]], dtype=float32)" ] }, "execution_count": 33, diff --git a/mlprodict/npy/onnx_numpy_wrapper.py b/mlprodict/npy/onnx_numpy_wrapper.py index cf6a5c234..878ff3b46 100644 --- a/mlprodict/npy/onnx_numpy_wrapper.py +++ b/mlprodict/npy/onnx_numpy_wrapper.py @@ -4,6 +4,7 @@ .. versionadded:: 0.6 """ +import warnings from .onnx_numpy_annotation import get_args_kwargs from .onnx_numpy_compiler import OnnxNumpyCompiler @@ -22,9 +23,10 @@ def append(self, name, cl): classes. """ if name in self.stored: - raise RuntimeError( # pragma: no cover - "Class %r already exists in\n%r\n---\n%r" % ( - name, ", ".join(sorted(self.stored)), cl)) + warnings.warn( # pragma: no cover + "Class %r overwritten in\n%r\n---\n%r" % ( + name, ", ".join(sorted(self.stored)), cl), + RuntimeWarning) self.stored[name] = cl globals()[name] = cl diff --git a/mlprodict/npy/onnx_sklearn_wrapper.py b/mlprodict/npy/onnx_sklearn_wrapper.py index 319a6f72d..914bc8026 100644 --- a/mlprodict/npy/onnx_sklearn_wrapper.py +++ b/mlprodict/npy/onnx_sklearn_wrapper.py @@ -288,7 +288,7 @@ def addattr(operator, obj): def _internal_decorator(fct, op_version=None, runtime=None, signature=None, - register_class=None): + register_class=None, overwrite=True, options=None): name = "onnxsklearn_parser_%s_%s_%s" % ( fct.__name__, str(op_version), runtime) newclass = type( @@ -305,12 +305,12 @@ def _internal_decorator(fct, op_version=None, runtime=None, signature=None, update_registered_converter_npy( register_class, "Sklearn%s" % getattr( register_class, "__name__", "noname"), - res, shape_fct=None, overwrite=False) + res, shape_fct=None, overwrite=overwrite, options=options) return res def onnxsklearn_transformer(op_version=None, runtime=None, signature=None, - register_class=None): + register_class=None, overwrite=True): """ Decorator to declare a converter for a transformer implemented using :epkg:`numpy` syntax but executed with :epkg:`ONNX` @@ -322,6 +322,7 @@ def onnxsklearn_transformer(op_version=None, runtime=None, signature=None, for transformer ``NDArraySameType("all")`` :param register_class: automatically register this converter for this class to :epkg:`sklearn-onnx` + :param overwrite: overwrite existing registered function if any .. versionadded:: 0.6 """ @@ -332,12 +333,13 @@ def decorator_fct(fct): return _internal_decorator(fct, signature=signature, op_version=op_version, runtime=runtime, - register_class=register_class) + register_class=register_class, + overwrite=overwrite) return decorator_fct def onnxsklearn_regressor(op_version=None, runtime=None, signature=None, - register_class=None): + register_class=None, overwrite=True): """ Decorator to declare a converter for a regressor implemented using :epkg:`numpy` syntax but executed with :epkg:`ONNX` @@ -349,6 +351,7 @@ def onnxsklearn_regressor(op_version=None, runtime=None, signature=None, for transformer ``NDArraySameType("all")`` :param register_class: automatically register this converter for this class to :epkg:`sklearn-onnx` + :param overwrite: overwrite existing registered function if any .. versionadded:: 0.6 """ @@ -359,12 +362,13 @@ def decorator_fct(fct): return _internal_decorator(fct, signature=signature, op_version=op_version, runtime=runtime, - register_class=register_class) + register_class=register_class, + overwrite=overwrite) return decorator_fct def onnxsklearn_classifier(op_version=None, runtime=None, signature=None, - register_class=None): + register_class=None, overwrite=True): """ Decorator to declare a converter for a classifier implemented using :epkg:`numpy` syntax but executed with :epkg:`ONNX` @@ -376,6 +380,7 @@ def onnxsklearn_classifier(op_version=None, runtime=None, signature=None, for transformer ``NDArraySameType("all")`` :param register_class: automatically register this converter for this class to :epkg:`sklearn-onnx` + :param overwrite: overwrite existing registered function if any .. versionadded:: 0.6 """ @@ -386,13 +391,16 @@ def decorator_fct(fct): return _internal_decorator(fct, signature=signature, op_version=op_version, runtime=runtime, - register_class=register_class) + register_class=register_class, + overwrite=overwrite, + options={'zipmap': [False, True, 'columns']}) return decorator_fct def _internal_method_decorator(register_class, method, op_version=None, runtime=None, signature=None, - method_names=None): + method_names=None, overwrite=True, + options=None): if isinstance(method_names, str): method_names = (method_names, ) @@ -412,6 +420,8 @@ def _internal_method_decorator(register_class, method, op_version=None, ("T:all", ), dtypes_out=((numpy.int64, ), 'T')) if method_names is None: method_names = ("predict", "predict_proba") + if options is None: + options = {'zipmap': [False, True, 'columns']} elif issubclass(register_class, ClusterMixin): if signature is None: signature = NDArrayType( @@ -480,12 +490,14 @@ def _check_(op): update_registered_converter_npy( register_class, "Sklearn%s" % getattr( register_class, "__name__", "noname"), - res, shape_fct=None, overwrite=False) + res, shape_fct=None, overwrite=overwrite, + options=options) return res def onnxsklearn_class(method_name, op_version=None, runtime=None, - signature=None, method_names=None): + signature=None, method_names=None, + overwrite=True): """ Decorator to declare a converter for a class derivated from :epkg:`scikit-learn`, implementing inference method @@ -501,6 +513,7 @@ def onnxsklearn_class(method_name, op_version=None, runtime=None, ONNX function :param method_names: if None, method names is guessed based on the class kind (transformer, regressor, classifier, clusterer) + :param overwrite: overwrite existing registered function if any .. versionadded:: 0.6 """ @@ -508,7 +521,8 @@ def decorator_class(objclass): _internal_method_decorator( objclass, method=getattr(objclass, method_name), signature=signature, op_version=op_version, - runtime=runtime, method_names=method_names) + runtime=runtime, method_names=method_names, + overwrite=overwrite) return objclass return decorator_class From 10c7f143b40ae46bfc90aad2b5859810a676bf84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Fri, 26 Mar 2021 10:19:36 +0100 Subject: [PATCH 08/10] add option nocl --- _unittests/ut_npy/test_custom_classifier.py | 21 +++++++++++++++------ mlprodict/npy/onnx_sklearn_wrapper.py | 6 ++++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/_unittests/ut_npy/test_custom_classifier.py b/_unittests/ut_npy/test_custom_classifier.py index b84d386e4..306d11257 100644 --- a/_unittests/ut_npy/test_custom_classifier.py +++ b/_unittests/ut_npy/test_custom_classifier.py @@ -29,6 +29,7 @@ def __init__(self): def fit(self, X, y=None, sample_weights=None): lr = LogisticRegression().fit(X, y, sample_weights) + self.classes_ = lr.classes_ self.coef_ = lr.coef_ # pylint: disable=W0201 self.intercept_ = lr.intercept_ # pylint: disable=W0201 if len(y.shape) == 1 or y.shape[1] == 1: @@ -101,6 +102,7 @@ def __init__(self): def fit(self, X, y=None, sample_weights=None): lr = LogisticRegression().fit(X, y, sample_weights) + self.classes_ = lr.classes_ self.coef_ = lr.coef_ # pylint: disable=W0201 self.intercept_ = lr.intercept_ # pylint: disable=W0201 if len(y.shape) == 1 or y.shape[1] == 1: @@ -133,7 +135,9 @@ def setUp(self): update_registered_converter( CustomLinearClassifier, "SklearnCustomLinearClassifier", custom_linear_classifier_shape_calculator, - custom_linear_classifier_converter) + custom_linear_classifier_converter, + options={'zipmap': [False, True, 'columns'], + 'nocl': [True, False]}) @ignore_warnings((DeprecationWarning, RuntimeWarning)) def test_function_classifier(self): @@ -142,7 +146,8 @@ def test_function_classifier(self): X.shape[0]).astype(numpy.float32)) >= 0).astype(numpy.int64) dec = CustomLinearClassifier() dec.fit(X, y) - onx = to_onnx(dec, X.astype(numpy.float32)) + onx = to_onnx(dec, X.astype(numpy.float32), + options={id(dec): {'zipmap': False}}) oinf = OnnxInference(onx) exp = dec.predict(X) prob = dec.predict_proba(X) @@ -157,7 +162,8 @@ def test_function_classifier3_float32(self): X.shape[0]).astype(numpy.float32)) >= 0).astype(numpy.int64) dec = CustomLinearClassifier3() dec.fit(X, y) - onx = to_onnx(dec, X.astype(numpy.float32)) + onx = to_onnx(dec, X.astype(numpy.float32), + options={id(dec): {'zipmap': False}}) oinf = OnnxInference(onx) exp = dec.predict(X) prob = dec.predict_proba(X) # pylint: disable=W0612 @@ -176,7 +182,8 @@ def test_function_classifier3_float64(self): X.shape[0]).astype(numpy.float32)) >= 0).astype(numpy.int64) dec = CustomLinearClassifier3() dec.fit(X, y) - onx = to_onnx(dec, X.astype(numpy.float64)) + onx = to_onnx(dec, X.astype(numpy.float64), + options={id(dec): {'zipmap': False}}) oinf = OnnxInference(onx) exp = dec.predict(X) prob = dec.predict_proba(X) @@ -199,7 +206,8 @@ def test_function_classifier_onnx_float32(self): self.assertNotEmpty(res) exp1 = dec.predict(X) # pylint: disable=E1101 prob1 = dec.predict_proba(X) # pylint: disable=E1101 - onx = to_onnx(dec, X.astype(numpy.float32)) + onx = to_onnx(dec, X.astype(numpy.float32), + options={id(dec): {'zipmap': False}}) oinf = OnnxInference(onx) exp2 = dec.predict(X) # pylint: disable=E1101 prob2 = dec.predict_proba(X) # pylint: disable=E1101 @@ -223,7 +231,8 @@ def test_function_classifier_onnx_float64(self): self.assertEqual(len(res), 2) exp1 = dec.predict(X) # pylint: disable=E1101 prob1 = dec.predict_proba(X) # pylint: disable=E1101 - onx = to_onnx(dec, X.astype(numpy.float64)) + onx = to_onnx(dec, X.astype(numpy.float64), + options={id(dec): {'zipmap': False}}) oinf = OnnxInference(onx) exp2 = dec.predict(X) # pylint: disable=E1101 prob2 = dec.predict_proba(X) # pylint: disable=E1101 diff --git a/mlprodict/npy/onnx_sklearn_wrapper.py b/mlprodict/npy/onnx_sklearn_wrapper.py index 914bc8026..e49465739 100644 --- a/mlprodict/npy/onnx_sklearn_wrapper.py +++ b/mlprodict/npy/onnx_sklearn_wrapper.py @@ -393,7 +393,8 @@ def decorator_fct(fct): runtime=runtime, register_class=register_class, overwrite=overwrite, - options={'zipmap': [False, True, 'columns']}) + options={'zipmap': [False, True, 'columns'], + 'nocl': [False, True]}) return decorator_fct @@ -421,7 +422,8 @@ def _internal_method_decorator(register_class, method, op_version=None, if method_names is None: method_names = ("predict", "predict_proba") if options is None: - options = {'zipmap': [False, True, 'columns']} + options = {'zipmap': [False, True, 'columns'], + 'nocl': [False, True]} elif issubclass(register_class, ClusterMixin): if signature is None: signature = NDArrayType( From 13cf182ea057d45d6bb0d6dee235b16c53e2cf41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Fri, 26 Mar 2021 10:37:23 +0100 Subject: [PATCH 09/10] fix one issue with conversion --- _unittests/ut_npy/test_custom_embedded_models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_unittests/ut_npy/test_custom_embedded_models.py b/_unittests/ut_npy/test_custom_embedded_models.py index 8e1cb8177..4a0ba334c 100644 --- a/_unittests/ut_npy/test_custom_embedded_models.py +++ b/_unittests/ut_npy/test_custom_embedded_models.py @@ -52,6 +52,7 @@ def fit(self, X, y, sample_weights=None): X[sign == 0], y[sign == 0]) self.lr1_ = LogisticRegression().fit( # pylint: disable=W0201 X[sign == 1], y[sign == 1]) + self.classes_ = self.lr0_.classes_ return self @@ -138,7 +139,8 @@ def common_test_function_classifier_embedded(self, dtype): X.shape[0]).astype(numpy.float32)) >= 0).astype(numpy.int64) dec = TwoLogisticRegressionOnnx() dec.fit(X, y) - onx = to_onnx(dec, X.astype(dtype)) + onx = to_onnx(dec, X.astype(dtype), + options={id(dec): {'zipmap': False}}) oinf = OnnxInference(onx) exp = dec.predict(X) # pylint: disable=E1101 prob = dec.predict_proba(X) # pylint: disable=E1101 From a1b5588bfbadf1ed6c66e7f67e1a99003f276ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Fri, 26 Mar 2021 16:27:27 +0100 Subject: [PATCH 10/10] fix unit test on cluster --- _unittests/ut_npy/test_custom_classifier.py | 4 +-- _unittests/ut_npy/test_custom_clusterer.py | 4 +-- .../ut_npy/test_custom_embedded_models.py | 2 +- mlprodict/npy/__init__.py | 2 +- mlprodict/npy/onnx_sklearn_wrapper.py | 29 +++++++++++++++++++ 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/_unittests/ut_npy/test_custom_classifier.py b/_unittests/ut_npy/test_custom_classifier.py index 306d11257..5b8657620 100644 --- a/_unittests/ut_npy/test_custom_classifier.py +++ b/_unittests/ut_npy/test_custom_classifier.py @@ -29,7 +29,7 @@ def __init__(self): def fit(self, X, y=None, sample_weights=None): lr = LogisticRegression().fit(X, y, sample_weights) - self.classes_ = lr.classes_ + self.classes_ = lr.classes_ # pylint: disable=W0201 self.coef_ = lr.coef_ # pylint: disable=W0201 self.intercept_ = lr.intercept_ # pylint: disable=W0201 if len(y.shape) == 1 or y.shape[1] == 1: @@ -102,7 +102,7 @@ def __init__(self): def fit(self, X, y=None, sample_weights=None): lr = LogisticRegression().fit(X, y, sample_weights) - self.classes_ = lr.classes_ + self.classes_ = lr.classes_ # pylint: disable=W0201 self.coef_ = lr.coef_ # pylint: disable=W0201 self.intercept_ = lr.intercept_ # pylint: disable=W0201 if len(y.shape) == 1 or y.shape[1] == 1: diff --git a/_unittests/ut_npy/test_custom_clusterer.py b/_unittests/ut_npy/test_custom_clusterer.py index 4cbc52601..a2570db9c 100644 --- a/_unittests/ut_npy/test_custom_clusterer.py +++ b/_unittests/ut_npy/test_custom_clusterer.py @@ -16,7 +16,7 @@ from skl2onnx.common.data_types import guess_numpy_type, Int64TensorType from mlprodict.onnx_conv import to_onnx from mlprodict.onnxrt import OnnxInference -from mlprodict.npy import onnxsklearn_classifier, onnxsklearn_class +from mlprodict.npy import onnxsklearn_cluster, onnxsklearn_class import mlprodict.npy.numpy_onnx_impl as nxnp @@ -67,7 +67,7 @@ class CustomCluster3(CustomCluster): pass -@onnxsklearn_classifier(register_class=CustomCluster3) +@onnxsklearn_cluster(register_class=CustomCluster3) def custom_cluster_converter3(X, op_=None): if X.dtype is None: raise AssertionError("X.dtype cannot be None.") diff --git a/_unittests/ut_npy/test_custom_embedded_models.py b/_unittests/ut_npy/test_custom_embedded_models.py index 4a0ba334c..9b76a89e0 100644 --- a/_unittests/ut_npy/test_custom_embedded_models.py +++ b/_unittests/ut_npy/test_custom_embedded_models.py @@ -52,7 +52,7 @@ def fit(self, X, y, sample_weights=None): X[sign == 0], y[sign == 0]) self.lr1_ = LogisticRegression().fit( # pylint: disable=W0201 X[sign == 1], y[sign == 1]) - self.classes_ = self.lr0_.classes_ + self.classes_ = self.lr0_.classes_ # pylint: disable=W0201 return self diff --git a/mlprodict/npy/__init__.py b/mlprodict/npy/__init__.py index fc6e47194..6cabc1930 100644 --- a/mlprodict/npy/__init__.py +++ b/mlprodict/npy/__init__.py @@ -13,4 +13,4 @@ from .onnx_sklearn_wrapper import ( update_registered_converter_npy, onnxsklearn_class, onnxsklearn_transformer, onnxsklearn_regressor, - onnxsklearn_classifier) + onnxsklearn_classifier, onnxsklearn_cluster) diff --git a/mlprodict/npy/onnx_sklearn_wrapper.py b/mlprodict/npy/onnx_sklearn_wrapper.py index e49465739..0244ece40 100644 --- a/mlprodict/npy/onnx_sklearn_wrapper.py +++ b/mlprodict/npy/onnx_sklearn_wrapper.py @@ -398,6 +398,35 @@ def decorator_fct(fct): return decorator_fct +def onnxsklearn_cluster(op_version=None, runtime=None, signature=None, + register_class=None, overwrite=True): + """ + Decorator to declare a converter for a cluster implemented using + :epkg:`numpy` syntax but executed with :epkg:`ONNX` + operators. + + :param op_version: :epkg:`ONNX` opset version + :param runtime: `'onnxruntime'` or one implemented by @see cl OnnxInference + :param signature: if None, the signature is replaced by a standard signature + for transformer ``NDArraySameType("all")`` + :param register_class: automatically register this converter + for this class to :epkg:`sklearn-onnx` + :param overwrite: overwrite existing registered function if any + + .. versionadded:: 0.6 + """ + if signature is None: + signature = NDArrayType(("T:all", ), dtypes_out=((numpy.int64, ), 'T')) + + def decorator_fct(fct): + return _internal_decorator(fct, signature=signature, + op_version=op_version, + runtime=runtime, + register_class=register_class, + overwrite=overwrite) + return decorator_fct + + def _internal_method_decorator(register_class, method, op_version=None, runtime=None, signature=None, method_names=None, overwrite=True,