Implements a function to measure the documentation

_unittests/ut_cli/test_cli_code.py

@@ -0,0 +1,36 @@
+"""
+@brief      test tree node (time=7s)
+"""
+import os
+import unittest
+from pyquickhelper.loghelper import BufferedPrint
+from pyquickhelper.pycode import ExtTestCase, get_temp_folder
+from pyquickhelper.__main__ import main
+
+
+class TestCliCodeHelper(ExtTestCase):
+
+    def test_code_stat_help(self):
+        st = BufferedPrint()
+        main(args=['code_stat', '--help'], fLOG=st.fprint)
+        res = str(st)
+        self.assertIn("usage: code_stat", res)
+
+    def test_code_stat_pyq(self):
+        st = BufferedPrint()
+        main(args=['code_stat', '-n', 'pyquickhelper'], fLOG=st.fprint)
+        res = str(st)
+        self.assertIn("value", res)
+
+    def test_code_stat_pyq_file(self):
+        temp = get_temp_folder(__file__, "temp_code_stat_pyq_file")
+        name = os.path.join(temp, "report.xlsx")
+        st = BufferedPrint()
+        main(args=['code_stat', '-n', 'pyquickhelper',
+             '-o', name], fLOG=st.fprint)
+        self.assertLess(len(str(st)), 10)
+        self.assertExists(name)
+
+
+if __name__ == "__main__":
+    unittest.main()

_unittests/ut_helpgen/test_notebooks_api.py

@@ -34,7 +34,7 @@ def test_convert_slides_api_html(self):
         try:
             res = nb2slides(nbr, outfile)
         except ReadUrlException as e:
-            warnings.warn(e)
+            warnings.warn(str(e))
             return
         self.assertGreater(len(res), 1)
         for r in res:

_unittests/ut_sphinxext/test_builders_missing.py

@@ -78,7 +78,10 @@ def get(self, name):  # pylint: disable=R1711
             inst._footnote = 'nofootnote'
             inst.rst_image_dest = ''
             if hasattr(inst, 'visit_table'):
-                inst.visit_table(element)
+                try:
+                    inst.visit_table(element)
+                except TypeError:
+                    pass
             for k in cl.__dict__:
                 if (k.startswith("visit_") or k.startswith("depart_") and
                         k not in ('visit_table', 'depart_table')):
@@ -110,7 +113,10 @@ def get(self, name):  # pylint: disable=R1711
                             raise Exception(
                                 f"Unable to run '{k}'") from e
             if hasattr(inst, 'depart_table'):
-                inst.depart_table(element)
+                try:
+                    inst.depart_table(element)
+                except TypeError:
+                    pass
 
 
 if __name__ == "__main__":

_unittests/ut_text/test_text_code_helper.py

@@ -1,14 +1,21 @@
 # -*- coding: utf-8 -*-
 """
-@brief      test tree node (time=1s)
+@brief      test tree node (time=6s)
 """
 
 
 import sys
 import os
 import unittest
-
-from pyquickhelper.texthelper import change_style, add_rst_links
+import numpy
+import pandas
+import pyquickhelper
+from pyquickhelper.sphinxext import sphinx_sharenet_extension
+from pyquickhelper.texthelper import change_style, add_rst_links, code_helper
+from pyquickhelper.texthelper.code_helper import (
+    measure_documentation,
+    measure_documentation_module)
+from pyquickhelper import texthelper
 
 
 class TestTextCodeHelper(unittest.TestCase):
@@ -28,6 +35,98 @@ def test_add_rst_links(self):
         res = add_rst_links(res, values)
         self.assertEqual(exp, res)
 
+    def test_measure_documentation(self):
+        res = measure_documentation(code_helper, include_hidden=True,
+                                    f_kind=lambda o: o.__name__)
+        self.assertIn("function", res)
+        c = res["function"]
+        expected = {
+            ("raw_length", "doc"): 0,
+            ("raw_length", "code"): 0,
+            ("length", "doc"): 0,
+            ("length", "code"): 0,
+            ("line", "doc"): 0,
+            ("line", "code"): 0,
+        }
+        for k, v in expected.items():
+            self.assertIn(k, c)
+            self.assertGreater(c[k], v + 1)
+        self.assertIn("measure_documentation", res)
+
+    def test_measure_documentation_df(self):
+        res = measure_documentation(
+            code_helper, include_hidden=True, as_df=True)
+        self.assertEqual(res.shape, (6, 4))
+        self.assertEqual(list(res.columns), [
+                         'kind', 'stat', 'doc_code', 'value'])
+
+    def test_measure_documentation2(self):
+        res = measure_documentation(sphinx_sharenet_extension, True)
+        self.assertIn("function", res)
+        c = res["function"]
+        expected = {
+            ("raw_length", "doc"): 0,
+            ("raw_length", "code"): 0,
+            ("length", "doc"): 0,
+            ("length", "code"): 0,
+            ("line", "doc"): 0,
+            ("line", "code"): 0,
+        }
+        for k, v in expected.items():
+            self.assertIn(k, c)
+            self.assertGreater(c[k], v + 1)
+        self.assertIn("class", res)
+        c = res["class"]
+        expected = {
+            ("raw_length", "doc"): 0,
+            ("raw_length", "code"): 0,
+            ("length", "doc"): 0,
+            ("length", "code"): 0,
+            ("line", "doc"): 0,
+            ("line", "code"): 0,
+        }
+        for k, v in expected.items():
+            self.assertIn(k, c)
+            self.assertGreater(c[k], v + 1)
+
+    def test_measure_documentation3(self):
+        res = measure_documentation_module(
+            texthelper, True, f_kind=lambda o: o.__name__)
+        self.assertIn("function", res)
+        c = res["function"]
+        expected = {
+            ("raw_length", "doc"): 0,
+            ("raw_length", "code"): 0,
+            ("length", "doc"): 0,
+            ("length", "code"): 0,
+            ("line", "doc"): 0,
+            ("line", "code"): 0,
+        }
+        for k, v in expected.items():
+            self.assertIn(k, c)
+            self.assertGreater(c[k], v + 1)
+
+    def test_measure_documentation_mod(self):
+        res = measure_documentation_module(pyquickhelper, True)
+        self.assertIn("function", res)
+        c = res["function"]
+        expected = {
+            ("raw_length", "doc"): 0,
+            ("raw_length", "code"): 0,
+            ("length", "doc"): 0,
+            ("length", "code"): 0,
+            ("line", "doc"): 0,
+            ("line", "code"): 0,
+        }
+        for k, v in expected.items():
+            self.assertIn(k, c)
+            self.assertGreater(c[k], v + 1)
+
+    def test_measure_documentation_numpy(self):
+        res = measure_documentation_module([numpy, pandas], True, as_df=True)
+        self.assertEqual(list(res.columns), [
+                         'module', 'kind', 'stat', 'doc_code', 'value'])
+
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main(verbosity=2)

src/pyquickhelper/__main__.py

@@ -31,6 +31,7 @@ def main(args, fLOG=print):
         from .filehelper.download_urls_helper import download_urls_in_folder_content
         from .cli.uvicorn_cli import uvicorn_app
         from .cli.profile_cli import profile_stat
+        from .cli.code_cli import code_stat
     except ImportError:  # pragma: no cover
         from pyquickhelper.cli.pyq_sync_cli import pyq_sync
         from pyquickhelper.cli.encryption_file_cli import encrypt_file, decrypt_file
@@ -49,6 +50,7 @@ def main(args, fLOG=print):
         from pyquickhelper.filehelper.download_urls_helper import download_urls_in_folder_content
         from pyquickhelper.cli.uvicorn_cli import uvicorn_app
         from pyquickhelper.cli.profile_cli import profile_stat
+        from pyquickhelper.cli.code_cli import code_stat
 
     fcts = dict(synchronize_folder=pyq_sync, encrypt_file=encrypt_file,
                 decrypt_file=decrypt_file, encrypt=encrypt,
@@ -61,7 +63,8 @@ def main(args, fLOG=print):
                 repeat_script=repeat_script,
                 ftp_upload=ftp_upload, set_password=set_password,
                 download_urls_in_folder_content=download_urls_in_folder_content,
-                uvicorn_app=uvicorn_app, profile_stat=profile_stat)
+                uvicorn_app=uvicorn_app, profile_stat=profile_stat,
+                code_stat=code_stat)
     return cli_main_helper(fcts, args=args, fLOG=fLOG)
 
 

src/pyquickhelper/cli/code_cli.py

@@ -0,0 +1,37 @@
+"""
+@file
+@brief Command line about transfering files.
+"""
+import importlib
+
+
+def code_stat(names, output=None, fLOG=print):
+    """
+    Returns statistics about the documentation of a module.
+
+    :param names: module name comma separated value
+    :param output: output file name
+    :param fLOG: logging function
+    :return: status
+
+    .. cmdref::
+        :title: Returns statistics about the documentation of a module
+        :cmd: -m pyquickhelper code_stat --help
+
+        The command line returns a table with the number of lines of code
+        and documentatation.
+    """
+    from ..texthelper.code_helper import measure_documentation_module
+    if not isinstance(names, list):
+        names = names.split(',')
+    mods = [importlib.import_module(name) for name in names]
+    stat = measure_documentation_module(mods, ratio=True, as_df=True)
+
+    if output is None:
+        fLOG(stat)
+        return None
+    if output.endswith(".xlsx"):
+        stat.to_excel(output, index=False)
+    else:
+        stat.to_csv(output, index=False)
+    return stat

src/pyquickhelper/helpgen/install_custom.py

@@ -99,11 +99,17 @@ def download_requirejs(to=".", fLOG=print,
     if location is None:
         from notebook import __file__ as local_location
         dirname = os.path.dirname(local_location)
-        location = os.path.join(
-            dirname, "static", "components", "requirejs", "require.js")
-        if not os.path.exists(location):
+        locations = [
+            os.path.join(
+                dirname, "static", "components", "requirejs", "require.js"),
+            os.path.join(
+                dirname, "..", "nbclassic", "static", "components", "requirejs",
+                "require.js")]
+        elocations = [loc for loc in locations if os.path.exists(loc)]
+        if len(elocations) == 0:
             raise FileNotFoundError(  # pragma: no cover
-                f"Unable to find requirejs in '{location}'")
+                f"Unable to find requirejs in '{locations}'")
+        location = elocations[0]
         shutil.copy(location, to)
         return [os.path.join(to, "require.js")]
     else:

src/pyquickhelper/helpgen/install_js_dep.py

@@ -4,10 +4,12 @@
 """
 
 import os
+import shutil
 from ..loghelper.flog import noLOG
 from .install_custom import download_revealjs, download_requirejs
 from ..filehelper import (
     synchronize_folder, change_file_status, download)
+from ..filehelper.internet_helper import ReadUrlException
 
 
 def install_javascript_tools(root, dest, fLOG=noLOG,
@@ -48,7 +50,12 @@ def install_javascript_tools(root, dest, fLOG=noLOG,
     # require.js
     expected = os.path.join(dest, "require.js")
     if not os.path.exists(expected):
-        one = download_requirejs(dest, fLOG=fLOG)
+        try:
+            one = download_requirejs(dest, fLOG=fLOG)
+        except ReadUrlException:
+            name = os.path.join(os.path.dirname(__file__), "require.js")
+            shutil.copy(name, expected)
+            one = [expected]
     else:
         one = [expected]
     lfiles.extend(one)
@@ -57,7 +64,12 @@ def install_javascript_tools(root, dest, fLOG=noLOG,
     expected = os.path.join(dest, "embed-amd.js")
     if not os.path.exists(expected):
         url = "https://unpkg.com/@jupyter-widgets/html-manager@0.20.0/dist/embed-amd.js"
-        one = [download(url, dest, fLOG=fLOG)]
+        try:
+            one = [download(url, dest, fLOG=fLOG)]
+        except ReadUrlException:
+            name = os.path.join(os.path.dirname(__file__), "embed-amd.js")
+            shutil.copy(name, expected)
+            one = [expected]
     else:
         one = [expected]
     lfiles.extend(one)

src/pyquickhelper/helpgen/sphinxm_convert_doc_sphinx_helper.py

@@ -1483,7 +1483,8 @@ def add_event(self, name):
         self._added_objects.append(('event', name))
         Sphinx.add_event(self, name)
 
-    def add_config_value(self, name, default, rebuild, types_=()):  # pylint: disable=W0221,W0237
+    def add_config_value(self, name, default, rebuild, types_=(), types=()):  # pylint: disable=W0221,W0237
+        types_ = types or types_
         self._added_objects.append(('config_value', name))
         Sphinx.add_config_value(self, name, default, rebuild, types_)