Merge conversion option in validate.py

_unittests/ut_onnxrt/test_onnxrt_validate_type.py

@@ -34,6 +34,9 @@ def filter_exp(cl, prob):
                 return '-64' in prob and '-cov' not in prob
             raise NotImplementedError(dtype)
 
+        def filter_scenario(m, p, o, e, e2):
+            return True
+
         if models is None:
             models = {
                 # 'DecisionTreeRegressor',
@@ -56,7 +59,8 @@ def filter_exp(cl, prob):
                 enumerate_validated_operator_opsets(
                     verbose, debug=True, fLOG=myfLOG, dump_folder=temp,
                     models=models, filter_exp=filter_exp,
-                    opset_min=11, store_models=True)):
+                    opset_min=11, store_models=True,
+                    filter_scenario=filter_scenario)):
 
             up = {}
             outputs = []

_unittests/ut_onnxrt/test_rt_valid_model_calibrated.py

@@ -28,7 +28,7 @@ def myprint(*args, **kwargs):
         rows = list(enumerate_validated_operator_opsets(
             verbose, models={"CalibratedClassifierCV"}, opset_min=11, fLOG=myprint,
             runtime='python', debug=True,
-            filter_scenario=lambda m, p, sc, ex: 'sgd' in sc))
+            filter_scenario=lambda m, p, sc, ex, ex2: 'sgd' in sc))
         self.assertGreater(len(rows), 1)
         self.assertGreater(len(buffer), 1)
         maxv = max(row['max_rel_diff_batch'] for row in rows)

_unittests/ut_onnxrt/test_rt_valid_model_gaussian_process.py

@@ -141,7 +141,7 @@ def test_rt_GaussianProcessRegressor_debug(self):
         def myprint(*args, **kwargs):
             buffer.append(" ".join(map(str, args)))
 
-        debug = False  # should be true
+        debug = True  # should be true
         rows = list(enumerate_validated_operator_opsets(
             verbose, models={"GaussianProcessRegressor"}, opset_min=11, fLOG=myprint,
             runtime='python', debug=debug,

_unittests/ut_onnxrt/test_rt_valid_model_gaussian_process_optim.py

@@ -30,15 +30,19 @@ def myprint(*args, **kwargs):
         rows = list(enumerate_validated_operator_opsets(
             verbose, models={"GaussianProcessRegressor"}, opset_min=11, fLOG=myprint,
             runtime='python', debug=debug,
-            filter_scenario=lambda m, p, s, e: p == "b-reg" and s == "rbf"))
+            filter_scenario=lambda m, p, s, e, e2: p == "b-reg" and s == "rbf"))
         self.assertGreater(len(rows), 1)
         self.assertGreater(len(buffer), 1 if debug else 0)
         opt = set(_.get('optim', '') for _ in rows)
-        exp = [{'', 'onnx-optim=cdist', 'optim=cdist', 'onnx'},
-               {'', 'onnx-optim=cdist', 'optim=cdist'}]
+        expcl = "<class 'sklearn.gaussian_process.gpr.GaussianProcessRegressor'>={'optim': 'cdist'}"
+        exp = [{'', 'onnx-' + expcl, expcl, 'onnx'},
+               {'', 'onnx-' + expcl, expcl}]
         self.assertIn(opt, exp)
         piv = summary_report(DataFrame(rows))
         opt = set(piv['optim'])
+        expcl = "cdist"
+        exp = [{'', 'onnx-' + expcl, expcl, 'onnx'},
+               {'', 'onnx-' + expcl, expcl}]
         self.assertIn(opt, exp)
 
 

_unittests/ut_onnxrt/test_rt_valid_model_gaussian_process_ort.py

@@ -9,6 +9,7 @@
 from pyquickhelper.texthelper.version_helper import compare_module_version
 from sklearn.exceptions import ConvergenceWarning
 from sklearn.utils.testing import ignore_warnings
+from sklearn.gaussian_process import GaussianProcessRegressor
 from sklearn.gaussian_process.kernels import RBF, ExpSineSquared
 from skl2onnx import __version__ as skl2onnx_version
 from skl2onnx.common.data_types import FloatTensorType
@@ -132,11 +133,19 @@ def test_rt_GaussianProcessRegressor_debug(self):
         def myprint(*args, **kwargs):
             buffer.append(" ".join(map(str, args)))
 
+        def filter_scenario(a, b, c, d, e):
+            if isinstance(e, dict) and GaussianProcessRegressor in e:
+                opt = e[GaussianProcessRegressor]
+                if opt.get('optim', '') == 'cdist':
+                    return False
+            return True
+
         debug = True
         rows = list(enumerate_validated_operator_opsets(
             verbose, models={"GaussianProcessRegressor"}, opset_min=11, fLOG=myprint,
             runtime='onnxruntime1', debug=debug,
-            filter_exp=lambda m, s: "reg-NSV" in s))
+            filter_exp=lambda m, s: "reg-NSV" in s,
+            filter_scenario=filter_scenario))
         self.assertGreater(len(rows), 1)
         self.assertGreater(len(buffer), 1 if debug else 0)
 
@@ -156,11 +165,19 @@ def test_rt_GaussianProcessRegressor_debug_std(self):
         def myprint(*args, **kwargs):
             buffer.append(" ".join(map(str, args)))
 
+        def filter_scenario(a, b, c, d, e):
+            if isinstance(e, dict) and GaussianProcessRegressor in e:
+                opt = e[GaussianProcessRegressor]
+                if opt.get('optim', '') == 'cdist':
+                    return False
+            return True
+
         debug = True
         rows = list(enumerate_validated_operator_opsets(
             verbose, models={"GaussianProcessRegressor"}, opset_min=11, fLOG=myprint,
             runtime='onnxruntime1', debug=debug,
-            filter_exp=lambda m, s: "b-reg-std-NSV" in s))
+            filter_exp=lambda m, s: "b-reg-std-NSV" in s,
+            filter_scenario=filter_scenario))
         self.assertGreater(len(rows), 1)
         self.assertGreater(len(buffer), 1 if debug else 0)
 

mlprodict/onnxrt/validate/validate.py

@@ -4,6 +4,7 @@
 The submodule relies on :epkg:`onnxconverter_common`,
 :epkg:`sklearn-onnx`.
 """
+import copy
 import numpy
 import sklearn
 from sklearn import __all__ as sklearn__all__, __version__ as sklearn_version
@@ -184,6 +185,22 @@ def _dictionary2str(di):
     return '/'.join(el)
 
 
+def _merge_options(all_conv_options, aoptions):
+    if not isinstance(aoptions, dict):
+        return copy.deepcopy(aoptions)
+    merged = {}
+    for k, v in all_conv_options.items():
+        if k in aoptions:
+            merged[k] = _merge_options(v, aoptions[k])
+        else:
+            merged[k] = copy.deepcopy(v)
+    for k, v in aoptions.items():
+        if k in all_conv_options:
+            continue
+        merged[k] = copy.deepcopy(v)
+    return merged
+
+
 def enumerate_compatible_opset(model, opset_min=9, opset_max=None,  # pylint: disable=R0914
                                check_runtime=True, debug=False,
                                runtime='python', dump_folder=None,
@@ -309,7 +326,8 @@ def enumerate_compatible_opset(model, opset_min=9, opset_max=None,  # pylint: di
                     new_conv_options = [{}]
 
                 if (filter_scenario is not None and
-                        not filter_scenario(model, prob, scenario, extra)):
+                        not filter_scenario(model, prob, scenario,
+                                            extra, new_conv_options)):
                     continue
 
                 if verbose >= 2 and fLOG is not None:
@@ -372,9 +390,15 @@ def enumerate_compatible_opset(model, opset_min=9, opset_max=None,  # pylint: di
                     for aoptions in new_conv_options:
                         obs_op = obs_op_0c.copy()
                         all_conv_options = {} if conv_options is None else conv_options.copy()
-                        all_conv_options.update(aoptions)
+                        all_conv_options = _merge_options(
+                            all_conv_options, aoptions)
                         obs_op['conv_options'] = all_conv_options
 
+                        if (filter_scenario is not None and
+                                not filter_scenario(model, prob, scenario,
+                                                    extra, all_conv_options)):
+                            continue
+
                         for rt in runtime:
                             def fct_conv(itt=inst, it=init_types[0][1], ops=opset,
                                          options=all_conv_options):