Improves backtest coverage, update documentation (#408)

* Improves backtest coverage
* Update azure-pipelines.yml
* add examples to the documentation
* add sphinx extension

_doc/examples/plot_op_reducesum.py

@@ -240,20 +240,6 @@ def torch_sum2(x, y):
 dfs.append(df)
 df.pivot("fct", "N", "average")
 
-###################################
-# Reduction on a particular case RKR
-# ++++++++++++++++++++++++++++++++++
-#
-# (N, 64, 16, 16), axis=(0, 2, 3)
-# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-axes = (0, 2, 3)
-df, piv, ax = benchmark_op(
-    axes, shape_fct=lambda dim: (dim, 64, 16, 16))
-dfs.append(df)
-df.pivot("fct", "N", "average")
-
-
 ###################################
 # Reduction on a particular case RKRK
 # +++++++++++++++++++++++++++++++++++

_doc/examples/plot_op_reducesumsquare.py

@@ -36,7 +36,7 @@
     from tensorflow.math import reduce_sum as tf_reduce_sum
     from tensorflow import convert_to_tensor
 except ImportError:
-    reduce_sum = None
+    tf_reduce_sum = None
 try:
     from torch import sum as torch_sum, from_numpy
 except ImportError:

_doc/examples/plot_profile.py

@@ -22,7 +22,7 @@
 from pyquickhelper.pycode.profiling import profile
 from mlprodict.onnx_conv import to_onnx
 from mlprodict.onnxrt import OnnxInference
-from mlprodict.onnxrt import get_ir_version
+from mlprodict import get_ir_version
 
 data = load_boston()
 X, y = data.data, data.target

_doc/sphinxdoc/source/conf.py

@@ -47,17 +47,15 @@
     'generate_automated_pages',
     'generate_visual_graphs',
     'generate_onnx_ops',
+    'mlprodict.npy.xop_sphinx',
 ])
 
 html_css_files = ['my-styles.css']
-
 html_logo = "phdoc_static/project_ico.png"
-
 html_sidebars = {}
-
 language = "en"
-
 mathdef_link_only = True
+onnx_doc_folder = os.path.join(os.path.dirname(__file__), 'onnxops')
 
 intersphinx_mapping.update({
     'cpyquickhelper': (

_doc/sphinxdoc/source/onnx.rst

@@ -16,6 +16,7 @@ predictions, it handles the graph and operators runtimes.
 .. toctree::
     :maxdepth: 1
 
+    onnxops/index
     onnx_runtime
     backends/index
 

_unittests/ut_asv_benchmark/test_create_asv_benchmark_histgbc.py

@@ -1,5 +1,5 @@
 """
-@brief      test log(time=3s)
+@brief      test log(time=16s)
 """
 import os
 import unittest

_unittests/ut_asv_benchmark/test_create_asv_benchmark_pyspy.py

@@ -1,5 +1,5 @@
 """
-@brief      test log(time=3s)
+@brief      test log(time=16s)
 """
 import os
 import unittest

_unittests/ut_module/test_code_style.py

@@ -29,6 +29,7 @@ def test_style_src(self):
                          "onnx_helper.py:8",  # a bug with python3.8
                          "__init__.py:1: R0401: Cyclic import",
                          "R0401: Cyclic import (mlprodict -> mlprodict.nb_helper",
+                         "c_compilation.py:1: R0401: Cyclic import (mlprodict.npy.xop ->",
                          ])
 
     def test_style_test(self):

_unittests/ut_npy/test_xop_doc.py

@@ -2,9 +2,11 @@
 @brief      test log(time=10s)
 """
 import unittest
-from pyquickhelper.pycode import ExtTestCase
+from pyquickhelper.pycode import ExtTestCase, get_temp_folder
 from mlprodict.npy.xop import _dynamic_class_creation, Xop
-from mlprodict.npy.xop_auto import get_rst_doc
+from mlprodict.npy.xop_auto import (
+    get_rst_doc, get_operator_schemas, get_onnx_example,
+    onnx_documentation_folder)
 
 
 class TestXopDoc(ExtTestCase):
@@ -26,6 +28,67 @@ def test_loading_factory(self):
         Add = Xop.Add
         self.assertEqual(Add.__name__, 'OnnxAdd')
 
+    def test_get_operator_schemas(self):
+        tr = get_operator_schemas('Transpose', domain='', version=13)
+        self.assertEqual(len(tr), 1)
+        self.assertEqual(tr[0].name, 'Transpose')
+        self.assertEqual(tr[0].domain, '')
+        self.assertEqual(tr[0].since_version, 13)
+        tr = get_operator_schemas('Transpose', domain='', version='last')
+        self.assertGreater(len(tr), 1)
+        tr = get_operator_schemas('Transpose', domain='', version=None)
+        self.assertGreater(len(tr), 2)
+        tr2 = get_operator_schemas('Transpose', domain=None, version=None)
+        self.assertEqual(len(tr), len(tr2))
+        self.assertGreater(tr[0].since_version, tr[1].since_version)
+
+    def test_onnxt_rst_transpose(self):
+        rst = get_rst_doc('Transpose', version=13)
+        self.assertIn("  tensor(int64),", rst)
+        self.assertIn(".. _l-onnx-op-transpose-13:", rst)
+        rstall = get_rst_doc('Transpose', version=None)
+        self.assertIn('Transpose - 13', rstall)
+        self.assertIn('Transpose - 1', rstall)
+        rstdiff = get_rst_doc('Transpose', version=None, diff=True)
+        self.assertIn('Transpose - 13', rstdiff)
+        self.assertIn('Transpose - 1', rstdiff)
+        self.assertIn('.. raw:: html', rstdiff)
+
+    def test_onnxt_get_example(self):
+        content = get_onnx_example('Transpose')
+        self.assertIsInstance(content, dict)
+        self.assertGreater(len(content), 2)
+        for v in content.values():
+            self.assertIn('expect(', v)
+
+    def test_onnxt_rst_transpose_example(self):
+        rst = get_rst_doc('Transpose', version=13, example=True)
+        self.assertIn('all_permutations', rst)
+        self.assertIn('Examples', rst)
+        self.assertIn('data = np.random.random_sample', rst)
+
+    def test_onnxt_rst_transpose_example_all(self):
+        rst = get_rst_doc('Transpose', example=True, version=None)
+        self.assertIn('all_permutations', rst)
+        self.assertIn('Examples', rst)
+        self.assertIn('data = np.random.random_sample', rst)
+        spl = rst.split('**Examples**')
+        if len(spl) > 2:
+            raise AssertionError(
+                "Too many example sections:\n%s" % rst)
+
+    def test_missing_examples(self):
+        res = get_onnx_example('tttt')
+        self.assertEqual({}, res)
+
+    def test_onnx_documentation_folder(self):
+        temp = get_temp_folder(__file__, 'temp_onnx_documentation_folder')
+        pages = onnx_documentation_folder(temp, ['Add', 'Transpose', 'TopK'])
+        self.assertGreater(len(pages), 3)
+        index = pages[-1]
+        self.assertEndsWith('index.rst', index)
+
 
 if __name__ == "__main__":
+    # TestXopDoc().test_onnx_documentation_folder()
     unittest.main(verbosity=2)