diff --git a/_doc/sphinxdoc/source/api/testing.rst b/_doc/sphinxdoc/source/api/testing.rst index c8dcc42cb..f34e86d67 100644 --- a/_doc/sphinxdoc/source/api/testing.rst +++ b/_doc/sphinxdoc/source/api/testing.rst @@ -28,6 +28,8 @@ Einsum .. autosignature:: mlprodict.testing.experimental_c.custom_einsum_double +.. autosignature:: mlprodict.testing.einsum_bench.einsum_benchmark + .. autosignature:: mlprodict.testing.einsum_impl_ext.numpy_diagonal .. autosignature:: mlprodict.testing.einsum_impl_ext.numpy_extended_dot diff --git a/_unittests/ut_cli/test_cli_einsum.py b/_unittests/ut_cli/test_cli_einsum.py new file mode 100644 index 000000000..6b6a671e4 --- /dev/null +++ b/_unittests/ut_cli/test_cli_einsum.py @@ -0,0 +1,77 @@ +""" +@brief test tree node (time=4s) +""" +import os +import unittest +from pyquickhelper.loghelper import BufferedPrint +from pyquickhelper.pycode import ExtTestCase, get_temp_folder +from mlprodict.__main__ import main + + +class TestCliEinsum(ExtTestCase): + + def test_cli_einsum(self): + st = BufferedPrint() + main(args=["einsum_test", "--help"], fLOG=st.fprint) + res = str(st) + self.assertIn("verbose", res) + + def test_cli_excel(self): + temp = get_temp_folder(__file__, "temp_cli_excel") + name = os.path.join(temp, "res.xlsx") + st = BufferedPrint() + main(args=["einsum_test", "--equation", "abc,cd->ad", + "--output", name, "--shape", "5", + "--verbose", "0"], fLOG=st.fprint) + self.assertExists(name) + res = str(st) + self.assertIn("wrote", res) + + def test_cli_csv(self): + temp = get_temp_folder(__file__, "temp_cli_csv") + name = os.path.join(temp, "res.csv") + st = BufferedPrint() + main(args=["einsum_test", "--equation", "abc,cd->ad", + "--output", name, "--shape", "(5,5,5);(5,5)", + "--verbose", "0"], fLOG=st.fprint) + self.assertExists(name) + res = str(st) + self.assertIn("wrote", res) + + def test_cli_csv_n(self): + temp = get_temp_folder(__file__, "temp_cli_csvn") + name = os.path.join(temp, "res.csv") + st = BufferedPrint() + main(args=["einsum_test", "--equation", "abc,cd->ad", + "--output", name, "--shape", "5,5", + "--verbose", "0"], fLOG=st.fprint) + self.assertExists(name) + res = str(st) + self.assertIn("wrote", res) + + def test_cli_csv_rt(self): + temp = get_temp_folder(__file__, "temp_cli_csv_rt") + name = os.path.join(temp, "res.csv") + st = BufferedPrint() + main(args=["einsum_test", "--equation", "abc,cd->ad", + "--output", name, "--shape", "(5,5,5);(5,5)", + "--verbose", "0", "--runtime", "onnxruntime"], + fLOG=st.fprint) + self.assertExists(name) + res = str(st) + self.assertIn("wrote", res) + + def test_cli_csv_perm(self): + temp = get_temp_folder(__file__, "temp_cli_csv_perm") + name = os.path.join(temp, "res.csv") + st = BufferedPrint() + main(args=["einsum_test", "--equation", "abc,cd->ad", + "--output", name, "--shape", "(5,5,5);(5,5)", + "--verbose", "0", "--perm", "1"], fLOG=st.fprint) + self.assertExists(name) + res = str(st) + self.assertIn("wrote", res) + + +if __name__ == "__main__": + unittest.main() diff --git a/_unittests/ut_testing/test_einsum.py b/_unittests/ut_testing/test_einsum.py index 31e22f585..32dc25b81 100644 --- a/_unittests/ut_testing/test_einsum.py +++ b/_unittests/ut_testing/test_einsum.py @@ -6,6 +6,7 @@ from contextlib import redirect_stdout import itertools import numpy +from onnx import numpy_helper from onnxruntime import ( InferenceSession, GraphOptimizationLevel, SessionOptions) from pyquickhelper.pycode import ExtTestCase @@ -628,7 +629,38 @@ def test_exc(self): r = repr(EinsumSubOp(2, 'transpose', 0, perm=(1, 0))) self.assertIn("EinsumSubOp('transpose', 0, perm=(1, 0))", r) + def test_bid_nd_bin(self): + + def local_test(inp1, inp2): + exp = numpy.einsum('bid,nd->bin', inp1, inp2) + seq = decompose_einsum_equation( + 'bid,nd->bin', clean=True, strategy='numpy') + got = apply_einsum_sequence(seq, inp1, inp2) + self.assertEqualArray(exp, got, decimal=3) + + onx = seq.to_onnx('Y', 'X1', 'X2') + oinf = OnnxInference(onx) + got = oinf.run({'X1': inp1, 'X2': inp2})['Y'] + self.assertEqualArray(exp, got, decimal=3) + + onx = seq.to_onnx( + 'Y', 'X1', 'X2', + initializer=[numpy_helper.from_array(inp2, name="X2")]) + oinf = OnnxInference(onx) + got = oinf.run({'X1': inp1})['Y'] + self.assertEqualArray(exp, got, decimal=3) + + inp1 = numpy.arange(2 * 3 * 5).reshape((2, 3, 5)).astype(numpy.float32) + inp2 = numpy.arange(5 * 7).reshape((5, 7)).astype(numpy.float32) + local_test(inp1, inp2.T) + + inp1 = numpy.random.uniform(size=[4, 5, 7]).astype(numpy.float32) + inp2 = numpy.random.uniform(size=[7, 8]).astype(numpy.float32) + local_test(inp1, inp2.T) + + self.optimize_compare('bid,nd->bin') + if __name__ == "__main__": - # TestEinsum().test_many_2() + # TestEinsum().test_bid_nd_bin() unittest.main() diff --git a/_unittests/ut_testing/test_einsum_benchmark.py b/_unittests/ut_testing/test_einsum_benchmark.py new file mode 100644 index 000000000..5c110fb9f --- /dev/null +++ b/_unittests/ut_testing/test_einsum_benchmark.py @@ -0,0 +1,37 @@ +""" +@brief test log(time=8s) +""" +import unittest +from pyquickhelper.pycode import ExtTestCase +from mlprodict.testing.einsum_bench import einsum_benchmark + + +class TestEinsumBenchmark(ExtTestCase): + + def test_benchmark1(self): + for rt in ['numpy', 'python', 'onnxruntime']: + with self.subTest(rt=rt): + res = list(einsum_benchmark(shape=5)) + self.assertEqual(len(res), 2) + + def test_benchmark2(self): + for rt in ['numpy', 'python', 'onnxruntime']: + with self.subTest(rt=rt): + res = list(einsum_benchmark(shape=[5, 6])) + self.assertEqual(len(res), 4) + + def test_benchmark1_shape(self): + for rt in ['numpy', 'python', 'onnxruntime']: + with self.subTest(rt=rt): + res = list(einsum_benchmark(shape=[(5, 5, 5), (5, 5)])) + self.assertEqual(len(res), 2) + + def test_benchmarkn(self): + for rt in ['numpy']: + with self.subTest(rt=rt): + res = list(einsum_benchmark(shape=5, perm=True)) + self.assertEqual(len(res), 48) + + +if __name__ == "__main__": + unittest.main() diff --git a/mlprodict/__main__.py b/mlprodict/__main__.py index 0072b638d..b38a90994 100644 --- a/mlprodict/__main__.py +++ b/mlprodict/__main__.py @@ -21,6 +21,7 @@ def main(args, fLOG=print): from .cli.asv_bench import asv_bench from .cli.asv2csv import asv2csv from .cli.replay import benchmark_replay + from .cli.einsum import einsum_test except ImportError: # pragma: no cover from mlprodict.cli.validate import validate_runtime from mlprodict.cli.convert_validate import convert_validate @@ -28,6 +29,7 @@ def main(args, fLOG=print): from mlprodict.cli.asv_bench import asv_bench from mlprodict.cli.asv2csv import asv2csv from mlprodict.cli.replay import benchmark_replay + from mlprodict.cli.einsum import einsum_test fcts = dict(validate_runtime=validate_runtime, convert_validate=convert_validate, @@ -35,7 +37,8 @@ def main(args, fLOG=print): onnx_stats=onnx_stats, asv_bench=asv_bench, asv2csv=asv2csv, - benchmark_replay=benchmark_replay) + benchmark_replay=benchmark_replay, + einsum_test=einsum_test) try: from pyquickhelper.cli import cli_main_helper except ImportError: # pragma: no cover diff --git a/mlprodict/cli/__init__.py b/mlprodict/cli/__init__.py index d9f059bb9..43fa491ba 100644 --- a/mlprodict/cli/__init__.py +++ b/mlprodict/cli/__init__.py @@ -3,5 +3,6 @@ @brief Shortcut to *cli*. """ from .convert_validate import convert_validate +from .einsum import einsum_test from .optimize import onnx_optim from .validate import validate_runtime diff --git a/mlprodict/cli/einsum.py b/mlprodict/cli/einsum.py new file mode 100644 index 000000000..6120eef52 --- /dev/null +++ b/mlprodict/cli/einsum.py @@ -0,0 +1,76 @@ +""" +@file +@brief Command line to check einsum scenarios. +""" +import os + + +def einsum_test(equation="abc,cd->abd", shape="30", perm=False, + runtime='python', verbose=1, fLOG=print, + output=None, number=5, repeat=5): + """ + Investigates whether or not the decomposing einsum is faster. + + :param equation: einsum equation to test + :param shape: an integer (all dimension gets the same size) or + a list of shapes in a string separated with `;`) or + a list of integer to try out multiple shapes, + example: `5`, `(5,5,5),(5,5)`, `5,6` + :param perm: check on permutation or all letter permutations + :param runtime: `'numpy'`, `'python'`, `'onnxruntime'` + :param verbose: verbose + :param fLOG: logging function + :param output: output file (usually a csv file or an excel file), + it requires pandas + :param number: usual parameter to measure a function + :param repeat: usual parameter to measure a function + + .. cmdref:: + :title: Investigates whether or not the decomposing einsum is faster. + :cmd: -m mlprodict einsum_test --help + :lid: l-cmd-einsum_test + + The command checks whether or not decomposing an einsum function + is faster than einsum implementation. + + Example:: + + python -m mlprodict einsum_test --equation="abc,cd->abd" --output=res.csv + """ + from ..testing.einsum_bench import einsum_benchmark # pylint: disable=E0402 + + perm = perm in ('True', '1', 1, True) + if "(" not in shape: + if "," in shape: + shape = list(map(int, shape.split(","))) + else: + shape = int(shape) + else: + shapes = shape.replace('(', '').replace(')', '').split(";") + shape = [] + for sh in shapes: + spl = sh.split(',') + shape.append(tuple(map(int, spl))) + verbose = int(verbose) + number = int(number) + repeat = int(repeat) + + res = einsum_benchmark(equation=equation, shape=shape, perm=perm, + runtime=runtime, use_tqdm=verbose > 0, + number=number, repeat=repeat) + if output not in ('', None): + import pandas + df = pandas.DataFrame(res) + ext = os.path.splitext(output)[-1] + if ext == '.csv': + df.to_csv(output, index=False) + fLOG('[einsum_test] wrote file %r.' % output) + elif ext == '.xlsx': + df.to_excel(output, index=False) + fLOG('[einsum_test] wrote file %r.' % output) + else: + raise ValueError( + "Unknown extension %r in file %r." % (ext, output)) + else: + for r in res: + fLOG(r) diff --git a/mlprodict/testing/bench_helper.py b/mlprodict/testing/bench_helper.py new file mode 100644 index 000000000..927be121a --- /dev/null +++ b/mlprodict/testing/bench_helper.py @@ -0,0 +1,48 @@ +""" +@file +@brief Helpers for benchmarks. +""" +from timeit import Timer +import numpy + + +def measure_time(stmt, *x, repeat=5, number=5, div_by_number=True, first_run=True): + """ + Measures a statement and returns the results as a dictionary. + + :param stmt: string + :param *x: inputs + :param repeat: average over *repeat* experiment + :param number: number of executions in one row + :param div_by_number: divide by the number of executions + :param first_run: if True, runs the function once before measuring + :return: dictionary + + See `Timer.repeat + `_ + for a better understanding of parameter *repeat* and *number*. + The function returns a duration corresponding to + *number* times the execution of the main statement. + """ + try: + stmt(*x) + except RuntimeError as e: # pragma: no cover + raise RuntimeError("{}-{}".format(type(x), x.dtype)) from e + + def fct(): + stmt(*x) + + if first_run: + fct() + tim = Timer(fct) + res = numpy.array(tim.repeat(repeat=repeat, number=number)) + total = numpy.sum(res) + if div_by_number: + res /= number + mean = numpy.mean(res) + dev = numpy.mean(res ** 2) + dev = max(0, (dev - mean**2)) ** 0.5 + mes = dict(average=mean, deviation=dev, min_exec=numpy.min(res), + max_exec=numpy.max(res), repeat=repeat, number=number, + total=total) + return mes diff --git a/mlprodict/testing/einsum_bench.py b/mlprodict/testing/einsum_bench.py new file mode 100644 index 000000000..53997f8e9 --- /dev/null +++ b/mlprodict/testing/einsum_bench.py @@ -0,0 +1,151 @@ +""" +@file +@brief Function to measure the performance of einsum decomposition. +""" +from itertools import permutations +import numpy +from onnx import helper, TensorProto +from onnxruntime import InferenceSession +from ..onnxrt import OnnxInference +from .bench_helper import measure_time +from .einsum_impl import decompose_einsum_equation, apply_einsum_sequence + + +def _make_einsum_model(equation, opset=13): + from skl2onnx.common._topology import OPSET_TO_IR_VERSION # pylint: disable=E0611,E0001 + inputs = equation.split('->')[0].split(',') + + model = helper.make_model( + opset_imports=[helper.make_operatorsetid('', opset)], + ir_version=OPSET_TO_IR_VERSION.get(opset, 7), + producer_name='mlprodict', + producer_version='0.1', + graph=helper.make_graph( + name='einsum_test', + inputs=[ + helper.make_tensor_value_info( + "X%d" % i, TensorProto.FLOAT, None) # pylint: disable=E1101 + for i in range(len(inputs))], + outputs=[ + helper.make_tensor_value_info( + "Y", TensorProto.FLOAT, None)], # pylint: disable=E1101 + nodes=[ + helper.make_node( + "Einsum", ["X%d" % i for i in range(len(inputs))], ["Y"], + equation=equation) + ] + ) + ) + return model + + +def _make_inputs(equation, shapes): + inputs = equation.split('->')[0].split(',') + dims = [len(i) for i in inputs] + + if isinstance(shapes, int): + N = shapes + shapes = [(N, ) * le for le in dims] + else: + if len(shapes) != len(inputs): + raise ValueError( + "Unexpected number of shapes %r with equation %r." + "" % (shapes, equation)) + inputs = [numpy.random.randn(*sh) for sh in shapes] + return [i.astype(numpy.float32) for i in inputs] + + +def einsum_benchmark(equation="abc,cd->abd", shape=30, perm=False, + runtime='python', use_tqdm=False, + number=5, repeat=5, opset=13): + """ + Investigates whether or not the decomposing einsum is faster. + + :param equation: einsum equation to test + :param shape: an integer (all dimension gets the same size) or + a list of shapes in a string separated with `;`) + :param perm: check on permutation or all letter permutations + :param runtime: numpy, python, onnxruntime + :param use_tqdm: show progress + :param output: output file (usually a csv file or an excel file), + it requires pandas + :param number: usual parameter to measure a function + :param repeat: usual parameter to measure a function + :param opset: target opset + :return: list of dictionaries as an iterator + """ + scenarios = [] + if (isinstance(shape, list) and + all(map(lambda t: isinstance(t, int), shape))): + shape_list = shape + else: + shape_list = [shape] + + if perm: + if equation.lower() != equation: + raise ValueError( + "Only equations with lower letters are allowed but equation %r " + "is not." % equation) + letters = list(sorted(set( + c for c in equation if "a" <= c < "z" or "A" <= c < "Z"))) + for p in permutations(letters): + replace = {d: c for c, d in zip(letters, p)} + eq = equation + for k, v in replace.items(): + eq = eq.replace(k, v.upper()) + eq = eq.lower() + for dec in ['einsum', 'dec']: + for sh in shape_list: + scenarios.append((eq, runtime, dec, sh)) + else: + for dec in ['einsum', 'dec']: + for sh in shape_list: + scenarios.append((equation, runtime, dec, sh)) + + if use_tqdm: + from tqdm import tqdm + loop = tqdm(scenarios) + else: + loop = scenarios + + for eq, rt, dec, sh in loop: + inputs = _make_inputs(equation, sh) + + if dec == 'dec': + seq = decompose_einsum_equation(eq, strategy='numpy', clean=True) + else: + seq = None + + if rt == 'numpy': + if dec == 'einsum': + fct = lambda *x, eq=eq: numpy.einsum(eq, *x, optimize=True) + else: + fct = lambda *x, seq=seq: apply_einsum_sequence(seq, *x) + elif rt == 'onnxruntime': + if dec == 'einsum': + onx = _make_einsum_model(equation, opset=opset) + else: + onx = seq.to_onnx('Y', *["X%d" % i for i in range(len(inputs))], + opset=opset) + sess = InferenceSession(onx.SerializeToString()) # pylint: disable=W0612 + fct = lambda *x, se=sess: se.run( + None, {"X%d" % i: v for i, v in enumerate(x)}) + elif rt == 'python': + if dec == 'einsum': + onx = _make_einsum_model(equation, opset=opset) + else: + onx = seq.to_onnx('Y', *["X%d" % i for i in range(len(inputs))], + opset=opset) + oinf = OnnxInference(onx) # pylint: disable=W0612 + fct = lambda *x, oi=oinf: oi.run( + {"X%d" % i: v for i, v in enumerate(x)}) + else: + raise ValueError("Unexpected runtime %r." % rt) + + res = measure_time(fct, *inputs, repeat=repeat, number=number) + res['rt'] = rt + res['dec'] = dec + res['eq'] = eq + res['shapes'] = ";".join( + map(str, [m.shape for m in inputs])).replace(' ', '') + yield res diff --git a/mlprodict/testing/einsum_impl_classes.py b/mlprodict/testing/einsum_impl_classes.py index 895cad73c..4a5ac2c9e 100644 --- a/mlprodict/testing/einsum_impl_classes.py +++ b/mlprodict/testing/einsum_impl_classes.py @@ -1323,8 +1323,18 @@ def to_onnx(self, output, *inputs, dtype=None, verbose=False, :param opset: desired opset, None for the last one :param verbose: display intermediate operators :param kwargs: additional parameter to use when building - the ONNX graph + the ONNX graph, list of supported parameters: + *name*, *ir_version*, *producer_name*, + *producer_version*, *initializer* :return: ONNX graph + + Not all graphs can be converted into ONNX. Only graphs produced + with `strategy='numpy'` can be converted otherwise the following + error shows up: + + :: + + NotImplementedError: to_onnx not implemented for 'matmul'. """ # inputs if opset is None: @@ -1348,6 +1358,8 @@ def to_onnx(self, output, *inputs, dtype=None, verbose=False, names = {i: name for i, name in enumerate(inputs)} nodes = [] inits = [] + if "initializer" in kwargs: + inits.extend(kwargs['initializer']) for op in self: for onx_node in op.to_onnx(names, verbose=verbose, opset=opset): if hasattr(onx_node, 'output'): diff --git a/mlprodict/testing/einsum_impl_ext.py b/mlprodict/testing/einsum_impl_ext.py index b0775fc4a..df905d02f 100644 --- a/mlprodict/testing/einsum_impl_ext.py +++ b/mlprodict/testing/einsum_impl_ext.py @@ -241,7 +241,7 @@ def numpy_extended_dot(m1, m2, axes, left, right, verbose=False): m1 = numpy.arange(8).reshape((2, 2, 2)) m2 = m1 + 10 - dot = numpy_extended_dot(m1, m2, [1], [0], [2], verbose=True)) + dot = numpy_extended_dot(m1, m2, [1], [0], [2], verbose=True) print(dot) The current implementation still uses :epkg:`numpy:einsum`