Refactors to improve profiling analysis (fix penalty, export onnx graphs)

_doc/examples/plot_orttraining_nn_gpu_fwbw_nesterov.py

@@ -2,8 +2,8 @@
 
 .. _l-orttraining-nn-gpu-fwbw-nesterov:
 
-Forward backward on a neural network on GPU (Nesterov)
-======================================================
+Forward backward on a neural network on GPU (Nesterov) and penalty
+==================================================================
 
 This example does the same as :ref:`l-orttraining-nn-gpu-fwbw`
 but updates the weights using `Nesterov momentum
@@ -18,6 +18,7 @@
 """
 import warnings
 import numpy
+import onnx
 from pandas import DataFrame
 from onnxruntime import get_device
 from sklearn.datasets import make_regression
@@ -26,11 +27,13 @@
 from sklearn.metrics import mean_squared_error
 from onnxcustom.plotting.plotting_onnx import plot_onnxs
 from mlprodict.onnx_conv import to_onnx
+from mlprodict.plotting.text_plot import onnx_simple_text_plot
 from onnxcustom.utils.orttraining_helper import get_train_initializer
 from onnxcustom.utils.onnx_helper import onnx_rename_weights
 from onnxcustom.training.optimizers_partial import (
     OrtGradientForwardBackwardOptimizer)
 from onnxcustom.training.sgd_learning_rate import LearningRateSGDNesterov
+from onnxcustom.training.sgd_learning_penalty import ElasticLearningPenalty
 
 
 X, y = make_regression(1000, n_features=10, bias=2)
@@ -92,11 +95,76 @@
 
 df = DataFrame({'ort losses': train_session.train_losses_,
                 'skl losses:': nn.loss_curve_})
-df.plot(title="Train loss against iterations", logy=True)
+df.plot(title="Train loss against iterations (Nesterov)", logy=True)
 
 ##############################################
 # The convergence rate is different but both classes
 # do not update the learning exactly the same way.
 
+##############################################
+# Penalty
+# +++++++
+#
+# Default parameters for MLPRegressor suggest to penalize weights
+# during training: `alpha=1e-4`.
+
+nn = MLPRegressor(hidden_layer_sizes=(10, 10), max_iter=100,
+                  solver='sgd', learning_rate_init=5e-5,
+                  n_iter_no_change=1000, batch_size=10, alpha=1e-4,
+                  momentum=0.9, nesterovs_momentum=True)
+
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore')
+    nn.fit(X_train, y_train)
+
+print(nn.loss_curve_)
+
+################################################
+# Let's do the same with onnxruntime.
+
+train_session = OrtGradientForwardBackwardOptimizer(
+    onx, device=device, verbose=1,
+    learning_rate=LearningRateSGDNesterov(1e-4, nesterov=True, momentum=0.9),
+    learning_penalty=ElasticLearningPenalty(l1=0, l2=1e-4),
+    warm_start=False, max_iter=100, batch_size=10)
+train_session.fit(X, y)
+
+#########################################
+# Let's see the weights.
+
+state_tensors = train_session.get_state()
+
+##########################################
+# And the loss.
+
+print(train_session.train_losses_)
+
+df = DataFrame({'ort losses': train_session.train_losses_,
+                'skl losses:': nn.loss_curve_})
+df.plot(title="Train loss against iterations (Nesterov + penalty)", logy=True)
+
+###########################################
+# All ONNX graphs
+# +++++++++++++++
+#
+# Method Method :meth:`save_onnx_graph
+# <onnxcustom.training._base.BaseOnnxClass.save_onnx_graph>`
+# can export all the ONNX graph used by the model on disk.
+
+
+def print_graph(d):
+    for k, v in sorted(d.items()):
+        if isinstance(v, dict):
+            print_graph(v)
+        else:
+            print("\n++++++", v.replace("\\", "/"), "\n")
+            with open(v, "rb") as f:
+                print(onnx_simple_text_plot(onnx.load(f)))
+
+
+all_files = train_session.save_onnx_graph('.')
+print_graph(all_files)
+
+
 # import matplotlib.pyplot as plt
 # plt.show()

_doc/sphinxdoc/source/tutorial_training/tutorial_6_training_partial.rst

@@ -107,6 +107,10 @@ And train losses:
 
     losses = train_session.train_losses_
 
+Method :meth:`save_onnx_graph
+<onnxcustom.training._base.BaseOnnxClass.save_onnx_graph>`
+exports all graphs used by a model. It can be saved on disk
+or just serialized in memory.
 Next examples show that in practice.
 
 Cache

_unittests/ut_training/test_optimizers_forward_backward.py

@@ -6,8 +6,11 @@
 import io
 import pickle
 import logging
-from pyquickhelper.pycode import ExtTestCase, ignore_warnings, skipif_appveyor
+from pyquickhelper.pycode import (
+    ExtTestCase, ignore_warnings, skipif_appveyor,
+    get_temp_folder)
 import numpy
+import onnx
 from onnx.helper import set_model_props
 from sklearn.datasets import make_regression
 from sklearn.model_selection import train_test_split
@@ -1105,12 +1108,139 @@ def test_ort_gradient_optimizers_use_numpy_pickle_w_nesterov_rate(self):
         self.assertGreater(len(losses), 1)
         self.assertFalse(any(map(numpy.isnan, losses)))
 
+    @unittest.skipIf(TrainingSession is None, reason="not training")
+    def test_ort_gradient_optimizers_nesterov_penalty_l2(self):
+        from onnxcustom.training.optimizers_partial import OrtGradientForwardBackwardOptimizer
+        X, y = make_regression(  # pylint: disable=W0632
+            100, n_features=10, bias=2, random_state=0)
+        X = X.astype(numpy.float32)
+        y = y.astype(numpy.float32)
+        w = (numpy.random.rand(y.shape[0]) + 1).astype(X.dtype)
+        X_train, _, y_train, __, w_train, ___ = train_test_split(
+            X, y, w)
+        reg = LinearRegression()
+        reg.fit(X_train, y_train, w_train)
+        reg.coef_ = reg.coef_.reshape((1, -1))
+        onx_model = to_onnx(reg, X_train, target_opset=opset,
+                            black_op={'LinearRegressor'})
+        set_model_props(onx_model, {'info': 'unit test'})
+        inits = ['coef', 'intercept']
+
+        train_session = OrtGradientForwardBackwardOptimizer(
+            onx_model, inits,
+            learning_rate=LearningRateSGDNesterov(
+                1e-4, nesterov=True, momentum=0.85),
+            learning_penalty=ElasticLearningPenalty(l1=0, l2=1e-4),
+            warm_start=False, max_iter=100, batch_size=10)
+
+        temp = get_temp_folder(
+            __file__, "temp_ort_gradient_optimizers_nesterov_penalty_l2")
+
+        saved = train_session.save_onnx_graph(temp)
+        saved_bytes = train_session.save_onnx_graph(bytes)
+        self.assertIsInstance(saved, dict)
+        self.assertNotEmpty(saved)
+        self.assertEqual(len(saved), len(saved_bytes))
+        checked = []
+        for k, v in saved_bytes.items():
+            if k == "learning_penalty":
+                for att, onxb in v.items():
+                    if att in ('penalty_grad_onnx_', 'penalty_onnx_'):
+                        onx = onnx.load(io.BytesIO(onxb))
+                        for init in onx.graph.initializer:  # pylint: disable=E1101
+                            vals = init.float_data
+                            if len(vals) == 1 and vals[0] == 0:
+                                checked.append((k, att))
+        if len(checked) != 2:
+            raise AssertionError("Unexpected parameter %r." % checked)
+        train_session.fit(X, y)
+
+        train_session = OrtGradientForwardBackwardOptimizer(
+            onx_model, inits, weight_name='weight',
+            learning_rate=LearningRateSGDNesterov(
+                1e-4, nesterov=True, momentum=0.9),
+            learning_penalty=ElasticLearningPenalty(l1=0, l2=1e-4),
+            warm_start=False, max_iter=100, batch_size=10)
+        temp = get_temp_folder(
+            __file__, "temp_ort_gradient_optimizers_nesterov_penalty_l2_weight")
+        train_session.save_onnx_graph(temp)
+        train_session.fit(X, y, w)
+
+    @unittest.skipIf(TrainingSession is None, reason="not training")
+    def test_ort_gradient_optimizers_nesterov_penalty_l1l2(self):
+        from onnxcustom.training.optimizers_partial import OrtGradientForwardBackwardOptimizer
+        X, y = make_regression(  # pylint: disable=W0632
+            100, n_features=10, bias=2, random_state=0)
+        X = X.astype(numpy.float32)
+        y = y.astype(numpy.float32)
+        w = (numpy.random.rand(y.shape[0]) + 1).astype(X.dtype)
+        X_train, _, y_train, __, w_train, ___ = train_test_split(
+            X, y, w)
+        reg = LinearRegression()
+        reg.fit(X_train, y_train, w_train)
+        reg.coef_ = reg.coef_.reshape((1, -1))
+        onx = to_onnx(reg, X_train, target_opset=opset,
+                      black_op={'LinearRegressor'})
+        set_model_props(onx, {'info': 'unit test'})
+        inits = ['coef', 'intercept']
+
+        train_session = OrtGradientForwardBackwardOptimizer(
+            onx, inits,
+            learning_rate=LearningRateSGDNesterov(
+                1e-4, nesterov=True, momentum=0.9),
+            learning_penalty=ElasticLearningPenalty(l1=1e-3, l2=1e-4),
+            warm_start=False, max_iter=100, batch_size=10)
+        train_session.fit(X, y)
+
+        train_session = OrtGradientForwardBackwardOptimizer(
+            onx, inits, weight_name='weight',
+            learning_rate=LearningRateSGDNesterov(
+                1e-4, nesterov=True, momentum=0.9),
+            learning_penalty=ElasticLearningPenalty(l1=1e-3, l2=1e-4),
+            warm_start=False, max_iter=100, batch_size=10)
+        train_session.fit(X, y, w)
+
+    @unittest.skipIf(TrainingSession is None, reason="not training")
+    def test_ort_gradient_optimizers_nesterov_penalty_l1l2_no(self):
+        from onnxcustom.training.optimizers_partial import OrtGradientForwardBackwardOptimizer
+        X, y = make_regression(  # pylint: disable=W0632
+            100, n_features=10, bias=2, random_state=0)
+        X = X.astype(numpy.float32)
+        y = y.astype(numpy.float32)
+        w = (numpy.random.rand(y.shape[0]) + 1).astype(X.dtype)
+        X_train, _, y_train, __, w_train, ___ = train_test_split(
+            X, y, w)
+        reg = LinearRegression()
+        reg.fit(X_train, y_train, w_train)
+        reg.coef_ = reg.coef_.reshape((1, -1))
+        onx = to_onnx(reg, X_train, target_opset=opset,
+                      black_op={'LinearRegressor'})
+        set_model_props(onx, {'info': 'unit test'})
+        inits = ['coef', 'intercept']
+
+        train_session = OrtGradientForwardBackwardOptimizer(
+            onx, inits,
+            learning_rate=LearningRateSGDNesterov(
+                1e-4, nesterov=False, momentum=0.9),
+            learning_penalty=ElasticLearningPenalty(l1=1e-3, l2=1e-4),
+            warm_start=False, max_iter=100, batch_size=10)
+        train_session.fit(X, y)
+
+        train_session = OrtGradientForwardBackwardOptimizer(
+            onx, inits, weight_name='weight',
+            learning_rate=LearningRateSGDNesterov(
+                1e-4, nesterov=False, momentum=0.9),
+            learning_penalty=ElasticLearningPenalty(l1=1e-3, l2=1e-4),
+            warm_start=False, max_iter=100, batch_size=10)
+        train_session.fit(X, y, w)
+
 
 if __name__ == "__main__":
     # import logging
     # logger = logging.getLogger('onnxcustom')
     # logger.setLevel(logging.DEBUG)
     # logging.basicConfig(level=logging.DEBUG)
-    # TestOptimizersForwardBackward().test_ort_gradient_optimizers_optimal_use_ort_w_elastic_penalty()
+    # cl = TestOptimizersForwardBackward()
+    # cl.test_ort_gradient_optimizers_nesterov_penalty_l2()
     # stop
     unittest.main()

_unittests/ut_utils/test_onnx_function.py

@@ -350,15 +350,15 @@ def fct(x):
 
         oinf = OnnxInference(onx)
         got = oinf.run({'loss': loss, 'W0': w1, 'W1': w2})
-        self.assertEqualArray(exp_loss, got['Y'], decimal=5)
+        self.assertEqualArray(exp_loss.reshape((-1, )), got['Y'], decimal=5)
 
         providers = device_to_providers('cpu')
         so = SessionOptions()
         so.log_severity_level = 4
         sess = InferenceSession(
             onx.SerializeToString(), so, providers=providers)
         got = sess.run(None, {'loss': loss, 'W0': w1, 'W1': w2})
-        self.assertEqualArray(exp_loss, got[0], decimal=5)
+        self.assertEqualArray(exp_loss.reshape((-1, )), got[0], decimal=5)
 
     def test_penalty_update(self):
         x = numpy.random.randn(10, 1).astype(numpy.float32)