Add example to look into neural network on GPU

.gitignore

@@ -5,6 +5,7 @@ bin/*
 dist/*
 build/*
 data/*
+.eggs/*
 *.jpg
 *.onnx
 *.pt
@@ -26,3 +27,5 @@ examples/model.onnx
 tests/model.onnx
 examples/onnxruntime_profile*.json
 version.txt
+_doc/bench/*.svg
+_doc/examples/*.svg

_doc/bench/bench_orttraining_nn_gpu.py

@@ -0,0 +1,130 @@
+"""
+
+.. _l-orttraining-nn-benchmark:
+
+Benchmark onnxruntime-training on a neural network
+==================================================
+
+You may profile the full example with on CPU with :epkg:`py-spy`:
+
+::
+
+    py-spy record -o bench_orttraining_nn_gpu.svg -r 10 --native -- python bench_orttraining_nn_gpu.py
+
+And with `nvprof` on GPU:
+
+::
+
+    nvprof -o bench_orttraining_nn_gpu.nvprof python bench_orttraining_nn_gpu.py --run_skl 0 --device cuda --opset 14
+
+.. contents::
+    :local:
+
+A neural network with scikit-learn
+++++++++++++++++++++++++++++++++++
+
+"""
+import warnings
+from pprint import pprint
+import time
+import numpy
+from pandas import DataFrame
+from onnxruntime import get_device
+from sklearn.datasets import make_regression
+from sklearn.model_selection import train_test_split
+from sklearn.neural_network import MLPRegressor
+from sklearn.metrics import mean_squared_error
+from mlprodict.onnx_conv import to_onnx
+from onnxcustom.training import add_loss_output, get_train_initializer
+from onnxcustom.training.optimizers import OrtGradientOptimizer
+
+
+def benchmark(N=1000, n_features=20, hidden_layer_sizes="25,25", max_iter=1000,
+              learning_rate_init=1e-4, batch_size=100, run_skl=True,
+              device='cpu', opset=14):
+    """
+    Compares :epkg:`onnxruntime-training` to :epkg:`scikit-learn` for
+    training. Training algorithm is SGD.
+
+    :param N: number of observations to train on
+    :param n_features: number of features
+    :param hidden_layer_sizes: hidden layer sizes, comma separated values
+    :param max_iter: number of iterations
+    :param learning_rate_init: initial learning rate
+    :param batch_size: batch size
+    :param run_skl: train scikit-learn in the same condition (True) or
+        just walk through one iterator with *scikit-learn*
+    :param device: `'cpu'` or `'cuda'`
+    :param opset: opset to choose for the conversion
+    """
+    N = int(N)
+    n_features = int(n_features)
+    max_iter = int(max_iter)
+    learning_rate_init = float(learning_rate_init)
+    batch_size = int(batch_size)
+    run_skl = run_skl in (1, True, '1', 'True')
+
+    print("N=%d" % N)
+    print("n_features=%d" % n_features)
+    print("hidden_layer_sizes=%s" % hidden_layer_sizes)
+    print("max_iter=%d" % max_iter)
+    print("learning_rate_init=%f" % learning_rate_init)
+    print("batch_size=%d" % batch_size)
+    print("run_skl=%r" % run_skl)
+    print("opset=%r" % opset)
+    print("device=%r" % device)
+    print('------------------')
+
+    hidden_layer_sizes = tuple(map(int, hidden_layer_sizes.split(",")))
+    X, y = make_regression(N, n_features=n_features, bias=2)
+    X = X.astype(numpy.float32)
+    y = y.astype(numpy.float32)
+    X_train, X_test, y_train, y_test = train_test_split(X, y)
+
+    nn = MLPRegressor(hidden_layer_sizes=hidden_layer_sizes,
+                      max_iter=max_iter if run_skl else 1,
+                      solver='sgd', learning_rate_init=learning_rate_init,
+                      n_iter_no_change=N, batch_size=batch_size)
+
+    begin = time.perf_counter()
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore')
+        nn.fit(X_train, y_train)
+    dur_skl = time.perf_counter() - begin
+
+    print("time_kl=%r, mean_squared_error=%r" % (
+        dur_skl, mean_squared_error(y_train, nn.predict(X_train))))
+
+    # conversion to ONNX
+    onx = to_onnx(nn, X_train[:1].astype(numpy.float32), target_opset=opset)
+
+    # add loss
+    onx_train = add_loss_output(onx)
+
+    # list of weights
+    inits = get_train_initializer(onx)
+    weights = {k: v for k, v in inits.items() if k != "shape_tensor"}
+
+    # training
+    print("device=%r get_device()=%r" % (device, get_device()))
+
+    #######################################
+    # The training session.
+
+    train_session = OrtGradientOptimizer(
+        onx_train, list(weights), device=device, verbose=0,
+        eta0=learning_rate_init,
+        warm_start=False, max_iter=max_iter, batch_size=batch_size)
+
+    begin = time.perf_counter()
+    train_session.fit(X, y)
+    dur_ort = time.perf_counter() - begin
+    print("time_kl=%r, mean_squared_error=%r" % (
+        dur_skl, mean_squared_error(y_train, nn.predict(X_train))))
+    print("time_ort=%r, last_trained_error=%r" % (
+        dur_ort, train_session.train_losses_[-1]))
+
+
+if __name__ == "__main__":
+    import fire
+    fire.Fire(benchmark)

_doc/examples/plot_catwoe_transformer.py

@@ -25,7 +25,7 @@
 <https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_iris.html>`_.
 Every feature is converter into integer.
 """
-import numpy as np
+import numpy
 from onnxruntime import InferenceSession
 from sklearn.datasets import load_iris
 from sklearn.preprocessing import OrdinalEncoder as SklOrdinalEncoder
@@ -40,8 +40,8 @@
 
 data = load_iris()
 X, y = data.data, data.target
-X = X.astype(np.int64)[:, :2]
-y = (y == 2).astype(np.int64)
+X = X.astype(numpy.int64)[:, :2]
+y = (y == 2).astype(numpy.int64)
 
 woe = WOEEncoder(cols=[0]).fit(X, y)
 print(woe.transform(X[:5]))
@@ -75,15 +75,15 @@ def ordenc_to_sklearn(op_mapping):
         mapping = column_map['mapping']
         res = []
         for i in range(mapping.shape[0]):
-            if np.isnan(mapping.index[i]):
+            if numpy.isnan(mapping.index[i]):
                 continue
             ind = mapping.iloc[i]
             while len(res) <= ind:
                 res.append(0)
             res[ind] = mapping.index[i]
-        cats[col] = np.array(res, dtype=np.int64)
+        cats[col] = numpy.array(res, dtype=numpy.int64)
 
-    skl_ord = SklOrdinalEncoder(categories=cats, dtype=np.int64)
+    skl_ord = SklOrdinalEncoder(categories=cats, dtype=numpy.int64)
     skl_ord.categories_ = cats
     return skl_ord
 
@@ -202,7 +202,7 @@ def woe_encoder_converter(scope, operator, container):
 
     sub = OnnxSubEstimator(op.ordinal_encoder, X,
                            op_version=opv)
-    cast = OnnxCast(sub, op_version=opv, to=np.float32)
+    cast = OnnxCast(sub, op_version=opv, to=numpy.float32)
     skl_ord = woeenc_to_sklearn(op.mapping)
     cat = OnnxSubEstimator(skl_ord, cast, op_version=opv,
                            output_names=operator.outputs[:1],

_doc/examples/plot_orttraining_linear_regression.py

@@ -18,7 +18,7 @@
 
 """
 from pprint import pprint
-import numpy as np
+import numpy
 from pandas import DataFrame
 from onnx import helper, numpy_helper, TensorProto
 from onnxruntime import (
@@ -31,8 +31,8 @@
 from tqdm import tqdm
 
 X, y = make_regression(n_features=2, bias=2)
-X = X.astype(np.float32)
-y = y.astype(np.float32)
+X = X.astype(numpy.float32)
+y = y.astype(numpy.float32)
 X_train, X_test, y_train, y_test = train_test_split(X, y)
 
 lr = LinearRegression()
@@ -78,8 +78,8 @@ def onnx_linear_regression(coefs, intercept):
     return model_def
 
 
-onx = onnx_linear_regression(lr.coef_.astype(np.float32),
-                             lr.intercept_.astype(np.float32))
+onx = onnx_linear_regression(lr.coef_.astype(numpy.float32),
+                             lr.intercept_.astype(numpy.float32))
 
 ########################################
 # Let's visualize it.
@@ -157,8 +157,9 @@ def onnx_linear_regression_training(coefs, intercept):
 
 
 onx_train = onnx_linear_regression_training(
-    np.random.randn(*lr.coef_.shape).astype(np.float32),
-    np.random.randn(*lr.intercept_.reshape((-1, )).shape).astype(np.float32))
+    numpy.random.randn(*lr.coef_.shape).astype(numpy.float32),
+    numpy.random.randn(
+        *lr.intercept_.reshape((-1, )).shape).astype(numpy.float32))
 
 plot_onnx(onx_train)
 
@@ -204,7 +205,7 @@ def __iter__(self):
         N = 0
         b = len(self) - self.batch_size
         while N < len(self):
-            i = np.random.randint(0, b)
+            i = numpy.random.randint(0, b)
             N += self.batch_size
             yield (self.X[i:i + self.batch_size],
                    self.y[i:i + self.batch_size])
@@ -304,7 +305,7 @@ def create_training_session(
 
 inputs = {'X': X_train[:1],
           'label': y_train[:1].reshape((-1, 1)),
-          'Learning_Rate': np.array([0.001], dtype=np.float32)}
+          'Learning_Rate': numpy.array([0.001], dtype=numpy.float32)}
 
 train_session.run(None, inputs)
 state_tensors = train_session.get_state()
@@ -315,7 +316,7 @@ def create_training_session(
 
 inputs = {'X': X_train[:1],
           'label': y_train[:1].reshape((-1, 1)),
-          'Learning_Rate': np.array([0.001], dtype=np.float32)}
+          'Learning_Rate': numpy.array([0.001], dtype=numpy.float32)}
 res = train_session.run(None, inputs)
 state_tensors = train_session.get_state()
 pprint(state_tensors)
@@ -379,7 +380,7 @@ def __init__(self, model_onnx, weights_to_train, loss_output_name='loss',
     def _init_learning_rate(self):
         self.eta0_ = self.eta0
         if self.learning_rate == "optimal":
-            typw = np.sqrt(1.0 / np.sqrt(self.alpha))
+            typw = numpy.sqrt(1.0 / numpy.sqrt(self.alpha))
             self.eta0_ = typw / max(1.0, (1 + typw) * 2)
             self.optimal_init_ = 1.0 / (self.eta0_ * self.alpha)
         else:
@@ -390,7 +391,7 @@ def _update_learning_rate(self, t, eta):
         if self.learning_rate == "optimal":
             eta = 1.0 / (self.alpha * (self.optimal_init_ + t))
         elif self.learning_rate == "invscaling":
-            eta = self.eta0_ / np.power(t + 1, self.power_t)
+            eta = self.eta0_ / numpy.power(t + 1, self.power_t)
         return eta
 
     def fit(self, X, y):
@@ -434,7 +435,7 @@ def _iteration(self, data_loader, learning_rate):
         :return: loss
         """
         actual_losses = []
-        lr = np.array([learning_rate], dtype=np.float32)
+        lr = numpy.array([learning_rate], dtype=numpy.float32)
         for batch_idx, (data, target) in enumerate(data_loader):
             if len(target.shape) == 1:
                 target = target.reshape((-1, 1))
@@ -444,7 +445,7 @@ def _iteration(self, data_loader, learning_rate):
                       self.input_names_[2]: lr}
             res = self.train_session_.run(None, inputs)
             actual_losses.append(res[self.loss_index_])
-        return np.array(actual_losses).mean()
+        return numpy.array(actual_losses).mean()
 
 ###########################################
 # Let's now train the model in a very similar way
@@ -456,7 +457,7 @@ def _iteration(self, data_loader, learning_rate):
 trainer.fit(X, y)
 print("training losses:", trainer.train_losses_)
 
-df = DataFrame({"iteration": np.arange(len(trainer.train_losses_)),
+df = DataFrame({"iteration": numpy.arange(len(trainer.train_losses_)),
                 "loss": trainer.train_losses_})
 df.set_index('iteration').plot(title="Training loss", logy=True)