add better handling of fLOG function for help generation

_unittests/ut_jenkinshelper/test_yaml.py

@@ -123,7 +123,7 @@ def test_jconvert_sequence_into_batch_file(self):
             OutputPrint=__name__ == "__main__")
 
         try:
-            self.a_test_jconvert_sequence_into_batch_file(sys.platform)
+            self.a_test_jconvert_sequence_into_batch_file("linux")
         except NotImplementedError as e:
             pass
         self.a_test_jconvert_sequence_into_batch_file("win")

_unittests/ut_pycode/test_full_unit_test_module_template.py

@@ -20,7 +20,7 @@
         sys.path.append(path)
     import src
 
-from src.pyquickhelper.loghelper.flog import fLOG, noLOG
+from src.pyquickhelper.loghelper.flog import fLOG
 from src.pyquickhelper.pycode import get_temp_folder, process_standard_options_for_setup, is_travis_or_appveyor
 from src.pyquickhelper.loghelper import git_clone
 
@@ -99,12 +99,14 @@ def skip_function(name, code, duration):
         fLOG("unit tests", root)
         for command in ["version", "write_version", "clean_pyd",
                         "setup_hook", "build_script", "copy27",
+                        "unittests -g .*ext.*",
                         "unittests", "unittests_LONG", "unittests_SKIP",
                         "build_sphinx"]:
             if command == "build_sphinx" and is_travis_or_appveyor():
                 # InkScape not installed for AppVeyor
                 continue
 
+            fLOG("#######################################################")
             fLOG("#######################################################")
             fLOG(command)
             fLOG("#######################################################")
@@ -115,20 +117,49 @@ def skip_function(name, code, duration):
             if command == "build_sphinx":
                 if thispath not in sys.path:
                     sys.path.append(thispath)
-                    fLOG("add", thispath)
+                    fLOG("UT add", thispath)
                     rem = True
+            log_lines = []
+
+            def logging_custom(*l, **p):
+                log_lines.append(l)
+            lcmd = command.split() if ' ' in command else [command]
+            stdout2 = StringIO()
+            stderr2 = StringIO()
+
             r = process_standard_options_for_setup(
-                [command], setup, "python3_module_template", module_name="project_name",
+                lcmd, setup, "python3_module_template", module_name="project_name",
                 port=8067, requirements=["pyquickhelper"], blog_list=blog_list,
-                fLOG=noLOG, additional_ut_path=[pyq, (root, True)],
+                fLOG=logging_custom, additional_ut_path=[pyq, (root, True)],
                 skip_function=skip_function, coverage_options={
                     "disable_coverage": True},
-                hook_print=False, stdout=stdout, stderr=stderr, use_run_cmd=True)
-            fLOG(r)
+                hook_print=False, stdout=stdout2, stderr=stderr2, use_run_cmd=True)
+            vout = stdout2.getvalue()
+            stdout.write(vout)
+            verr = stderr2.getvalue()
+            stderr.write(verr)
+            if "unittests" in command:
+                if not r:
+                    raise Exception("{0}-{1}".format(r, command))
+                for line in log_lines:
+                    fLOG("  ", line)
+                if len(log_lines) == 0:
+                    raise Exception(
+                        "command={0}\nOUT:\n{1}\nERR:\n{2}".format(command, vout, verr))
+                if "-e" in command and "running test   1, ut_module/test_convert_notebooks.py" in vout:
+                    raise Exception(vout)
+                if "-e" in command and "_ext" not in vout:
+                    raise Exception(vout)
+                if "LONG" in command and "running test   1, ut_module/test_convert_notebooks.py" in vout:
+                    raise Exception(vout)
+                if "LONG" not in command and "LONG" in vout:
+                    raise Exception(vout)
             if rem:
                 del sys.path[sys.path.index(thispath)]
             os.environ["PYTHONPATH"] = PYTHONPATH
 
+        fLOG("#######################################################")
+        fLOG("#######################################################")
         fLOG("OUT:\n", stdout.getvalue())
         fLOG("ERR:\n", stderr.getvalue())
 

src/pyquickhelper/filehelper/synchelper.py

@@ -93,21 +93,10 @@ def explore_folder_iterfile_repo(folder, log=fLOG):
         yield file
 
 
-def synchronize_folder(p1,
-                       p2,
-                       hash_size=1024 ** 2,
-                       repo1=False,
-                       repo2=False,
-                       size_different=True,
-                       no_deletion=False,
-                       filter=None,
-                       filter_copy=None,
-                       avoid_copy=False,
-                       operations=None,
-                       file_date=None,
-                       log1=False,
-                       copy_1to2=False,
-                       fLOG=fLOG):
+def synchronize_folder(p1, p2, hash_size=1024 ** 2, repo1=False, repo2=False,
+                       size_different=True, no_deletion=False, filter=None,
+                       filter_copy=None, avoid_copy=False, operations=None,
+                       file_date=None, log1=False, copy_1to2=False, fLOG=fLOG):
     """
     synchronize two folders (or copy if the second is empty), it only copies more recent files.
 
@@ -172,7 +161,7 @@ def filter_copy(file):
         Parameter *fLOG* was added.
     """
 
-    fLOG("form ", p1)
+    fLOG("from ", p1)
     fLOG("to   ", p2)
 
     if file_date is not None and not os.path.exists(file_date):

src/pyquickhelper/helpgen/install_js_dep.py

@@ -40,7 +40,7 @@ def install_javascript_tools(root, dest, fLOG=noLOG,
             folder = os.path.dirname(sphinxjp.themes.revealjs.__file__)
             js = os.path.join(folder, "templates", "revealjs", "static")
             os.mkdir(rev)
-            sync = synchronize_folder(js, rev, copy_1to2=True)
+            sync = synchronize_folder(js, rev, copy_1to2=True, fLOG=fLOG)
             fulls = [s[1].fullname for s in sync]
             change_file_status(rev)
             lfiles = fulls

src/pyquickhelper/helpgen/process_notebooks.py

@@ -47,7 +47,7 @@
 
 def process_notebooks(notebooks, outfold, build, latex_path=None, pandoc_path=None,
                       formats=("ipynb", "html", "python", "rst",
-                               "slides", "pdf", "present")):
+                               "slides", "pdf", "present"), fLOG=fLOG):
     """
     Converts notebooks into html, rst, latex, pdf, python, docx using
     `nbconvert <http://ipython.org/ipython-doc/rel-1.0.0/interactive/nbconvert.html>`_.
@@ -58,6 +58,7 @@ def process_notebooks(notebooks, outfold, build, latex_path=None, pandoc_path=No
     @param      pandoc_path path to pandoc
     @param      formats     list of formats to convert into (pdf format means latex then compilation)
     @param      latex_path  path to the latex compiler
+    @param      fLOG        logging function
     @return                 list of tuple *[(file, created or skipped)]*
 
     This function relies on `pandoc <http://johnmacfarlane.net/pandoc/index.html>`_.
@@ -109,7 +110,7 @@ def process_notebooks(notebooks, outfold, build, latex_path=None, pandoc_path=No
     """
     return _process_notebooks_in(notebooks=notebooks, outfold=outfold, build=build,
                                  latex_path=latex_path, pandoc_path=pandoc_path,
-                                 formats=formats)
+                                 formats=formats, fLOG=fLOG)
 
 
 def _process_notebooks_in_private(fnbcexe, list_args, options_args):
@@ -147,7 +148,8 @@ def _process_notebooks_in_private_cmd(fnbcexe, list_args, options_args, fLOG):
 
 def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_path=None,
                           formats=("ipynb", "html", "python", "rst",
-                                   "slides", "pdf", "present")):
+                                   "slides", "pdf", "present"),
+                          fLOG=fLOG):
     from nbconvert.nbconvertapp import main as nbconvert_main
     if pandoc_path is None:
         pandoc_path = find_pandoc_path()
@@ -313,7 +315,7 @@ def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_pat
                 # docutils if format is slides, not sure about the others
                 if nbconvert_main != fnbcexe or format not in {"slides"}:
                     out, err = _process_notebooks_in_private(
-                        fnbcexe, list_args, options_args)
+                        fnbcexe, list_args, options_args, fLOG=fLOG)
                 else:
                     # conversion into slides alter Jinja2 environment
                     # jinja2.exceptions.TemplateNotFound: rst

src/pyquickhelper/helpgen/sphinx_helper.py

@@ -8,7 +8,7 @@
 import sys
 
 from ..filehelper import synchronize_folder, explore_folder_iterfile
-from ..loghelper.flog import noLOG
+from ..loghelper.flog import noLOG, fLOG
 
 if sys.version_info[0] == 2:
     from codecs import open
@@ -23,7 +23,7 @@ def everything_but_python(fullname):
     return os.path.splitext(fullname)[-1] not in [".py", ".pyc"]
 
 
-def sphinx_add_scripts(source, dest, filter=everything_but_python):
+def sphinx_add_scripts(source, dest, filter=everything_but_python, fLOG=fLOG):
     """
     copy additional scripts to a folder for sphinx documentation
 
@@ -36,7 +36,7 @@ def sphinx_add_scripts(source, dest, filter=everything_but_python):
         os.makedirs(dest)
 
     res = synchronize_folder(
-        source, dest, repo1=False, repo2=False, filter=filter)
+        source, dest, repo1=False, repo2=False, filter=filter, fLOG=fLOG)
     return res
 
 

src/pyquickhelper/helpgen/sphinx_main.py

@@ -70,7 +70,8 @@ def generate_help_sphinx(project_var_name, clean=False, root=".",
                          extra_ext=None,
                          nbformats=("ipynb", "slides", "html", "python",
                                     "rst", "pdf", "present"),
-                         layout=[("html", "build", {}), ],  # ("epub", "build", {})],
+                         # ("epub", "build", {})],
+                         layout=[("html", "build", {}), ],
                          module_name=None, from_repo=True, add_htmlhelp=False,
                          copy_add_ext=None, direct_call=False, fLOG=fLOG):
     """
@@ -242,7 +243,6 @@ def generate_help_sphinx(project_var_name, clean=False, root=".",
 
         When there are too many notebooks, the notebook index is difficult to read.
 
-
     .. todoext::
         :title: replace command line by direct call to sphinx, nbconvert, nbpresent
         :tag: enhancement
@@ -255,7 +255,7 @@ def generate_help_sphinx(project_var_name, clean=False, root=".",
         It does not require to get script location.
         Not enough stable from virtual environment.
     """
-    setup_environment_for_help()
+    setup_environment_for_help(fLOG=fLOG)
     # we keep a clean list of modules
     # sphinx configuration is a module and the function loads and unloads it
     list_modules_start = set(sys.modules.keys())
@@ -439,7 +439,7 @@ def lay_build_override_newconf(t3):
     #########
     chan = os.path.join(root, "_doc", "sphinxdoc", "source", "filechanges.rst")
     generate_changes_repo(
-        chan, root, filter_commit=filter_commit, exception_if_empty=from_repo)
+        chan, root, filter_commit=filter_commit, exception_if_empty=from_repo, fLOG=fLOG)
 
     ######################################
     # we copy javascript dependencies, reveal.js
@@ -470,22 +470,18 @@ def lay_build_override_newconf(t3):
     ####################
     try:
 
-        prepare_file_for_sphinx_help_generation(
-            {},
-            root,
-            os.path.join(root, "_doc", "sphinxdoc", "source"),
-            subfolders=[
-                ("src/" + module_name, module_name),
-            ],
-            silent=True,
-            rootrep=("_doc.sphinxdoc.source.%s." % (module_name,), ""),
-            optional_dirs=optional_dirs,
-            mapped_function=mapped_function,
-            replace_relative_import=False,
-            module_name=module_name,
-            copy_add_ext=copy_add_ext,
-            use_sys=use_sys,
-            fLOG=fLOG)
+        prepare_file_for_sphinx_help_generation({}, root,
+                                                os.path.join(
+                                                    root, "_doc", "sphinxdoc", "source"),
+                                                subfolders=[
+                                                    ("src/" + module_name, module_name), ],
+                                                silent=True,
+                                                rootrep=("_doc.sphinxdoc.source.%s." % (
+                                                    module_name,), ""),
+                                                optional_dirs=optional_dirs, mapped_function=mapped_function,
+                                                replace_relative_import=False, module_name=module_name,
+                                                copy_add_ext=copy_add_ext, use_sys=use_sys,
+                                                fLOG=fLOG)
 
     except ImportErrorHelpGen as e:
 
@@ -552,7 +548,7 @@ def lay_build_override_newconf(t3):
                 os.mkdir(notebook_doc)
             nbs_all = process_notebooks(notebooks, build=build, outfold=notebook_doc,
                                         formats=nbformats, latex_path=latex_path,
-                                        pandoc_path=pandoc_path)
+                                        pandoc_path=pandoc_path, fLOG=fLOG)
             nbs_all = set(_[0]
                           for _ in nbs_all if os.path.splitext(_[0])[-1] == ".rst")
             if len(nbs_all) != len(indexlistnote):
@@ -577,7 +573,8 @@ def lay_build_override_newconf(t3):
     #############################################
     fLOG("~~~~ blog placeholder")
     if plist is not None:
-        replace_placeholder_by_recent_blogpost(all_tocs, plist, "__INSERT__")
+        replace_placeholder_by_recent_blogpost(
+            all_tocs, plist, "__INSERT__", fLOG=fLOG)
 
     #################################
     #  run the documentation generation
@@ -834,7 +831,7 @@ def lay_build_override_newconf(t3):
             fLOG("copy javascript static files from",
                  html_static_path, "to", builddoc)
             copy = synchronize_folder(
-                html_static_path, builddoc, copy_1to2=True)
+                html_static_path, builddoc, copy_1to2=True, fLOG=fLOG)
             fLOG("javascript", len(copy), "files copied")
         else:
             fLOG("[revealjs] no need, no folder", builddoc)

src/pyquickhelper/helpgen/sphinx_main_helper.py

@@ -38,7 +38,7 @@
 """
 
 
-def setup_environment_for_help():
+def setup_environment_for_help(fLOG=fLOG):
     """
     modifies environment variables to be able to use external tools
     such as `Inkscape <https://inkscape.org/>`_
@@ -172,7 +172,8 @@ def to_str(x):
 def generate_changes_repo(chan,
                           source,
                           exception_if_empty=True,
-                          filter_commit=lambda c: c.strip() != "documentation"):
+                          filter_commit=lambda c: c.strip() != "documentation",
+                          fLOG=fLOG):
     """
     Generates a rst tables containing the changes stored by a svn or git repository,
     the outcome is stored in a file.
@@ -182,6 +183,7 @@ def generate_changes_repo(chan,
     @param          source              source folder to get changes for
     @param          exception_if_empty  raises an exception if empty
     @param          filter_commit       function which accepts a commit to show on the documentation (based on the comment)
+    @param          fLOG                logging function
     @return                             string (rst tables with the changes)
 
     .. versionchanged:: 1.0
@@ -340,14 +342,15 @@ def compile_latex_output_final(root, latex_path, doall, afile=None, latex_book=F
                 raise HelpGenException(err)
 
 
-def replace_placeholder_by_recent_blogpost(all_tocs, plist, placeholder, nb_post=5):
+def replace_placeholder_by_recent_blogpost(all_tocs, plist, placeholder, nb_post=5, fLOG=fLOG):
     """
     replaces a place holder by a list of blog post
 
     @param      all_tocs        list of files to look into
     @param      plist           list of blog post
     @param      placeholder     place holder to replace
     @param      nb_post         number of blog post to display
+    @param      fLOG            logging function
     """
     def make_link(post):
         name = os.path.splitext(os.path.split(post.FileName)[-1])[0]