Fixes for onnx==1.11

.gitignore

@@ -319,3 +319,4 @@ cache-*.pickle
 onnxruntime*.json
 *net*.tar*
 _unittests/unittests.out
+mlprodict/npy/_cache/*.rst

_unittests/ut_onnx_conv/test_onnxrt_runtime_lightgbm.py

@@ -15,7 +15,8 @@
     BooleanTensorType, DoubleTensorType)
 from mlprodict.onnxrt import OnnxInference
 from mlprodict.onnx_conv import register_converters, to_onnx
-from mlprodict.tools.asv_options_helper import get_ir_version_from_onnx
+from mlprodict.tools.asv_options_helper import (
+    get_ir_version_from_onnx, get_last_opset)
 
 
 class TestOnnxrtRuntimeLightGbm(ExtTestCase):
@@ -349,7 +350,7 @@ def test_onnxrt_python_lightgbm_categorical_iris_dataframe(self):
         booster = lgb_train(params, train_data)
         exp = booster.predict(X_test)
 
-        onx = to_onnx(booster, df_train)
+        onx = to_onnx(booster, df_train, target_opset=get_last_opset())
         self.assertIn('ZipMap', str(onx))
 
         oif = OnnxInference(onx)
@@ -436,7 +437,8 @@ def test_missing_values(self):
             n_estimators=1, learning_rate=1)
         regressor.fit(_X_train, _y)
         regressor_onnx = to_onnx(
-            regressor, initial_types=_INITIAL_TYPES, rewrite_ops=True)
+            regressor, initial_types=_INITIAL_TYPES, rewrite_ops=True,
+            target_opset=get_last_opset())
         y_pred = regressor.predict(_X_test)
         y_pred_onnx = self._predict_with_onnx(regressor_onnx, _X_test)
         self._assert_almost_equal(
@@ -466,7 +468,8 @@ def test_missing_values_rf(self):
             n_estimators=10, bagging_freq=1, bagging_fraction=0.5)
         regressor.fit(_X_train, _y)
         regressor_onnx = to_onnx(
-            regressor, initial_types=_INITIAL_TYPES, rewrite_ops=True)
+            regressor, initial_types=_INITIAL_TYPES, rewrite_ops=True,
+            target_opset=get_last_opset())
         y_pred = regressor.predict(_X_test)
         y_pred_onnx = self._predict_with_onnx(regressor_onnx, _X_test)
         self._assert_almost_equal(
@@ -525,7 +528,7 @@ def test_objective(self):
                 regressor.fit(_X, _Y)
                 regressor_onnx = to_onnx(
                     regressor, initial_types=initial_types,
-                    rewrite_ops=True)
+                    rewrite_ops=True, target_opset=get_last_opset())
                 y_pred = regressor.predict(_X)
                 y_pred_onnx = self._predict_with_onnx(regressor_onnx, _X)
                 self._assert_almost_equal(
@@ -558,7 +561,7 @@ def test_objective_boosting_rf(self):
                 regressor.fit(_X, _Y)
                 regressor_onnx = to_onnx(
                     regressor, initial_types=initial_types,
-                    rewrite_ops=True)
+                    rewrite_ops=True, target_opset=get_last_opset())
                 y_pred = regressor.predict(_X)
                 y_pred_onnx = self._predict_with_onnx(regressor_onnx, _X) / 10
                 self._assert_almost_equal(
@@ -619,7 +622,8 @@ def test_lgbm_regressor(self):
 
         # float split
         onx = to_onnx(reg, X_train, options={'split': 10},
-                      rewrite_ops=True)
+                      rewrite_ops=True,
+                      target_opset=get_last_opset())
         oinf = OnnxInference(onx)
         got2 = oinf.run({'X': X_test})['variable']
         self.assertEqualArray(expected, got2, decimal=5)

_unittests/ut_onnx_conv/test_onnxrt_runtime_lightgbm_bug.py

@@ -6,9 +6,11 @@
 from logging import getLogger
 import numpy
 from pyquickhelper.pycode import ExtTestCase, skipif_circleci
+from pyquickhelper.texthelper.version_helper import compare_module_version
 from skl2onnx.common.data_types import FloatTensorType
 from mlprodict.onnxrt import OnnxInference
 from mlprodict.onnx_conv import register_converters, to_onnx
+from mlprodict.tools.asv_options_helper import get_last_opset
 
 
 class TestOnnxrtRuntimeLightGbmBug(ExtTestCase):
@@ -29,6 +31,12 @@ def setUp(self):
     @skipif_circleci('stuck')
     @unittest.skipIf(sys.platform == 'darwin', 'stuck')
     def test_xgboost_regressor(self):
+        try:
+            from onnxmltools import __version__
+        except ImportError:
+            return
+        if compare_module_version(__version__, '1.11') < 0:
+            return
         from xgboost import XGBRegressor
         try:
             from onnxmltools.convert import convert_xgboost
@@ -94,6 +102,12 @@ def test_missing_values(self):
     @skipif_circleci('stuck')
     @unittest.skipIf(sys.platform == 'darwin', 'stuck')
     def test_lightgbm_regressor(self):
+        try:
+            from onnxmltools import __version__
+        except ImportError:
+            return
+        if compare_module_version(__version__, '1.11') < 0:
+            return
         from lightgbm import LGBMRegressor
         try:
             from onnxmltools.convert import convert_lightgbm
@@ -155,9 +169,11 @@ def test_lightgbm_regressor_double(self):
                     learning_rate=0.0000001)
                 model.fit(X, y)
                 expected = model.predict(X)
-                model_onnx = to_onnx(model, X, rewrite_ops=True)
-                model_onnx2 = to_onnx(model, X.astype(numpy.float64),
-                                      rewrite_ops=True)
+                model_onnx = to_onnx(
+                    model, X, rewrite_ops=True, target_opset=get_last_opset())
+                model_onnx2 = to_onnx(
+                    model, X.astype(numpy.float64), rewrite_ops=True,
+                    target_opset=get_last_opset())
 
                 for i, mo in enumerate([model_onnx, model_onnx2]):
                     for rt in ['python', 'onnxruntime1']:

_unittests/ut_onnxrt/test_rt_valid_model_isolationforest.py

@@ -5,6 +5,7 @@
 from logging import getLogger
 from pyquickhelper.loghelper import fLOG
 from pyquickhelper.pycode import ExtTestCase
+from pyquickhelper.texthelper.version_helper import compare_module_version
 from sklearn.exceptions import ConvergenceWarning
 try:
     from sklearn.utils._testing import ignore_warnings
@@ -17,6 +18,8 @@
 class TestRtValidateIsolationForest(ExtTestCase):
 
     @ignore_warnings(category=(UserWarning, ConvergenceWarning, RuntimeWarning))
+    @unittest.skipIf(compare_module_version(skl2onnx_version, '1.11') < 0,
+                     reason="converter issue")
     def test_rt_IsolationForest_python(self):
         fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")
         logger = getLogger('skl2onnx')

mlprodict/onnx_conv/convert.py

@@ -362,7 +362,8 @@ def to_onnx(model, X=None, name=None, initial_types=None,
                     type(model)))
         return model.to_onnx(
             X=X, name=name, options=options, black_op=black_op,
-            white_op=white_op, final_types=final_types)
+            white_op=white_op, final_types=final_types,
+            target_opset=target_opset)
         # verbose=verbose)
 
     if rewrite_ops:

mlprodict/onnx_conv/operator_converters/conv_xgboost.py

@@ -218,17 +218,18 @@ def convert(scope, operator, container):
 
         # add nodes
         if dtype == numpy.float64:
-            container.add_node('TreeEnsembleRegressorDouble', operator.input_full_names,
-                               operator.output_full_names,
-                               name=scope.get_unique_operator_name(
-                                   'TreeEnsembleRegressorDouble'),
-                               op_domain='mlprodict', **attr_pairs)
+            container.add_node(
+                'TreeEnsembleRegressorDouble', operator.input_full_names,
+                operator.output_full_names,
+                name=scope.get_unique_operator_name(
+                    'TreeEnsembleRegressorDouble'),
+                op_domain='mlprodict', **attr_pairs)
         else:
-            container.add_node('TreeEnsembleRegressor', operator.input_full_names,
-                               operator.output_full_names,
-                               name=scope.get_unique_operator_name(
-                                   'TreeEnsembleRegressor'),
-                               op_domain='ai.onnx.ml', **attr_pairs)
+            container.add_node(
+                'TreeEnsembleRegressor', operator.input_full_names,
+                operator.output_full_names,
+                name=scope.get_unique_operator_name('TreeEnsembleRegressor'),
+                op_domain='ai.onnx.ml', **attr_pairs)
 
 
 class XGBClassifierConverter(XGBConverter):

mlprodict/onnxrt/validate/validate.py

@@ -17,7 +17,7 @@
 from ...tools.ort_wrapper import onnxrt_version
 from ...tools.model_info import analyze_model, set_random_state
 from ...tools.asv_options_helper import (
-    get_opset_number_from_onnx, get_ir_version_from_onnx)
+    get_opset_number_from_onnx, get_ir_version_from_onnx, get_last_opset)
 from ..onnx_inference import OnnxInference
 from ...onnx_tools.optim.sklearn_helper import inspect_sklearn_model, set_n_jobs
 from ...onnx_tools.optim.onnx_helper import onnx_statistics
@@ -471,7 +471,12 @@ def _call_conv_runtime_opset(
             for rt in runtime:
                 def fct_conv(itt=inst, it=init_types[0][1], ops=opset,
                              options=all_conv_options):
-                    return to_onnx(itt, it, target_opset=ops, options=options,
+                    if isinstance(ops, int):
+                        ops_dict = get_last_opset().copy()
+                        ops_dict[''] = ops
+                    else:
+                        ops_dict = ops
+                    return to_onnx(itt, it, target_opset=ops_dict, options=options,
                                    rewrite_ops=rt in ('', None, 'python',
                                                       'python_compiled'))
 

mlprodict/sklapi/onnx_tokenizer.py

@@ -15,6 +15,7 @@
     from onnxruntime_extensions import get_library_path
 except ImportError:
     get_library_path = None
+from mlprodict.tools.asv_options_helper import get_opset_number_from_onnx
 
 
 class SentencePieceTokenizerTransformer(BaseEstimator, TransformerMixin):
@@ -125,7 +126,7 @@ def _create_model(model_b64, domain='ai.onnx.contrib', opset=None):
                 mkv('out0', TensorProto.INT32, [None]),
                 mkv('out1', TensorProto.INT64, [None])])
         if opset is None:
-            opset = onnx_opset_version()
+            opset = min(get_opset_number_from_onnx(), onnx_opset_version())
         model = helper.make_model(graph, opset_imports=[
             helper.make_operatorsetid('', opset)])
         model.opset_import.extend([helper.make_operatorsetid(domain, 1)])
@@ -237,7 +238,7 @@ def _create_model(vocab, merges, padding_length,
                 mkv('input_ids', TensorProto.INT64, [None, None]),
                 mkv('attention_mask', TensorProto.INT64, [None, None])])
         if opset is None:
-            opset = onnx_opset_version()
+            opset = min(get_opset_number_from_onnx(), onnx_opset_version())
         model = helper.make_model(graph, opset_imports=[
             helper.make_operatorsetid('', opset)])
         model.opset_import.extend([helper.make_operatorsetid(domain, 1)])

mlprodict/tools/asv_options_helper.py

@@ -107,6 +107,19 @@ def get_opset_number_from_onnx(benchmark=True):
     return onnx_opset_version()
 
 
+def get_last_opset(ml=True):
+    """
+    Returns the last supported opset.
+
+    :param ml: includes domain `ai.onnx.ml`
+    :return: int or dictionary
+    """
+    if ml:
+        return {'': get_opset_number_from_onnx(),
+                'ai.onnx.ml': 2}
+    return get_opset_number_from_onnx()
+
+
 def get_ir_version_from_onnx(benchmark=True):
     """
     Retuns the current :epkg:`onnx` :epkg:`IR_VERSION`

requirements.txt

@@ -48,7 +48,7 @@ wheel
 xgboost
 
 # onnx
-onnx>=1.10.1
+onnx>=1.11
 onnxruntime>=1.10.0
 onnxruntime-extensions>=0.4.2
 skl2onnx>=1.10.2