Fixes #328, improves links in notebooks

_unittests/ut_helpgen/test_LONG_full_documentation_module_template.py

@@ -141,7 +141,7 @@ def __len__(self):
                 fLOG("[test_full_documentation] **********************************")
 
                 direct_call = i % 2 == 0
-                layout = ["pdf", "html"]
+                layout = ["html"]
 
                 logger1 = getLogger("docassert")
                 logger2 = getLogger("tocdelay")
@@ -296,6 +296,23 @@ def __len__(self):
                     raise Exception(
                         "Unable to find '{0}' in\n{1}".format(tofind, content))
 
+                # notebook links
+                files = [os.path.join(root, "_doc", "sphinxdoc", "build", "html",
+                                      "notebooks", "custom_notebooks.html"),
+                         os.path.join(root, "_doc", "sphinxdoc", "build", "html",
+                                      "notebooks", "custom_notebooks.slides.html")
+                         ]
+                for name in files:
+                    with open(name, "r", encoding="utf-8") as f:
+                        content = f.read()
+                    if "https://unpkg.com/@jupyter-widgets/html-manager@%5E0.20.0/dist/embed-amd.js" in content:
+                        raise AssertionError("Absolute link in %r." % name)
+                    if "https://cdnjs.cloudflare" in content:
+                        raise AssertionError(
+                            "Absolute cloudflare link in %r." % name)
+                    if "reveal.js/dist" in content:
+                        raise AssertionError("Wrong link in slides %r." % name)
+
             # final check
             logs = os.path.join(temp, "log_custom_000.txt")
             with open(logs, "r", encoding='utf-8') as f:

_unittests/ut_helpgen/test_LONG_full_documentation_module_template_rst.py

@@ -24,14 +24,15 @@ def test_full_documentation_module_template_rst(self):
         """
         This test might fail in sphinx-gallery due to a very long filename.
         Please look into the following commit:
-        https://github.com/sdpython/sphinx-gallery/commit/3ae9f13250cf25c75e1b17b2fade98b7a9940b0d.
+        https://github.com/sdpython/sphinx-gallery/commit/
+        3ae9f13250cf25c75e1b17b2fade98b7a9940b0d.
         """
         fLOG(
             __file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")
 
-        if is_travis_or_appveyor() in ('travis', 'appveyor') or sys.version_info[0] == 2:
+        if is_travis_or_appveyor() in ('travis', 'appveyor'):
             # travis fails due to the following:
             #       sitep = [_ for _ in site.getsitepackages() if "packages" in _]
             # AttributeError: 'module' object has no attribute

src/pyquickhelper/filehelper/compression_helper.py

@@ -298,7 +298,9 @@ def zip7_files(filename_7z, file_set, fLOG=noLOG, temp_folder="."):
     out, err = run_cmd(cmd, wait=True)
     if "Error:" in out or not os.path.exists(filename_7z):
         raise FileException(  # pragma: no cover
-            "An error occurred with cmd: '{0}'\nOUT:\n{1}\nERR\n{2}\n----".format(cmd, out, err))
+            "An error occurred with cmd: '{0}'\n"
+            "--OUT--\n{1}\n--ERR--\n{2}\n----".format(
+                cmd, out, err))
     return len(file_set)
 
 
@@ -462,7 +464,9 @@ def unrar_files(zipf, where_to=None, fLOG=noLOG, fvalid=None, remove_space=True)
         out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
         if len(err) > 0 or "Error:" in out:
             raise FileException(
-                "Unable to unrar file '{0}'\nOUT\n{1}\nERR\n{2}".format(zipf, out, err))
+                "Unable to unrar file '{0}'\n"
+                "--OUT--\n{1}\n--ERR--\n{2}".format(
+                    zipf, out, err))
 
         return explore_folder(where_to)[1]
     else:

src/pyquickhelper/helpgen/install_custom.py

@@ -63,10 +63,9 @@ def download_revealjs(temp_folder=".", unzip_to=".", fLOG=print,
         for r in res:
             if os.path.isdir(r):
                 continue
-            if ".gitignore" in r or ".travis.yml" in r or "index.html" in r \
-               or ".appveyor.yml" in r \
-               or "requirement" in r \
-               or "README" in r or "CONTRIBUTING.md" in r:
+            if (".gitignore" in r or ".travis.yml" in r or "index.html" in r or
+                ".appveyor.yml" in r or "requirement" in r or "README" in r or
+                    "CONTRIBUTING.md" in r):
                 os.remove(r)
             elif "/test/" in r.replace("\\", "/"):
                 os.remove(r)

src/pyquickhelper/helpgen/install_js_dep.py

@@ -6,7 +6,8 @@
 import os
 from ..loghelper.flog import noLOG
 from .install_custom import download_revealjs, download_requirejs
-from ..filehelper import synchronize_folder, change_file_status
+from ..filehelper import (
+    synchronize_folder, change_file_status, download)
 
 
 def install_javascript_tools(root, dest, fLOG=noLOG,
@@ -34,7 +35,6 @@ def install_javascript_tools(root, dest, fLOG=noLOG,
     else:
         rev = os.path.join(dest, "reveal.js")
         if not os.path.exists(rev):
-
             folder = os.path.dirname(revealjs.__file__)
             js = os.path.join(folder, "templates", "revealjs", "static")
             os.mkdir(rev)
@@ -52,4 +52,13 @@ def install_javascript_tools(root, dest, fLOG=noLOG,
     else:
         one = [expected]
     lfiles.extend(one)
+
+    # embed-ams.js
+    expected = os.path.join(dest, "embed-amd.js")
+    if not os.path.exists(expected):
+        url = "https://unpkg.com/@jupyter-widgets/html-manager@0.20.0/dist/embed-amd.js"
+        one = [download(url, dest, fLOG=fLOG)]
+    else:
+        one = [expected]
+    lfiles.extend(one)
     return lfiles

src/pyquickhelper/helpgen/pandoc_helper.py

@@ -19,8 +19,10 @@ def call_pandoc(params, fLOG=noLOG):
     cmd = '"{0}" {1}'.format(pandoc, params)
     out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
     if err is not None and "Cannot decode byte" in err:
-        raise Exception(
-            "Issue with pandac:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
+        raise RuntimeError(
+            "Issue with pandoc:\n{0}\n"
+            "--OUT--\n{1}\n--ERR--\n{2}".format(
+                cmd, out, err))
     return out, err
 
 

src/pyquickhelper/helpgen/post_process.py

@@ -179,6 +179,8 @@ def post_process_latex_output(root, doall, latex_book=False, exc=True,
     """
     if os.path.isfile(root):
         file = root
+        if fLOG:
+            fLOG("[post_process_latex_output] clean %r" % file)
         with open(file, "r", encoding="utf8") as f:
             content = f.read()
         with open(file + ".tex1~", "w", encoding="utf8") as f:
@@ -224,6 +226,8 @@ def post_process_python_output(root, doall, exc=True, nblinks=None, fLOG=None, n
     """
     if os.path.isfile(root):
         file = root
+        if fLOG:
+            fLOG("[post_process_python_output] clean %r" % file)
         with open(file, "r", encoding="utf8") as f:
             content = f.read()
         content = post_process_python(
@@ -301,7 +305,7 @@ def post_process_rst_output(file, html, pdf, python, slides, is_notebook=False,
     and checks that audio is only included in :epkg:`HTML`.
     """
     if fLOG:
-        fLOG("[post_process_rst_output] %r" % file)
+        fLOG("[post_process_rst_output] clean %r" % file)
 
     name = os.path.split(file)[1]
     noext = os.path.splitext(name)[0]
@@ -455,6 +459,9 @@ def startss(line):
                     docname = selected[0]
                     path = docname[len(folder):]
                     break
+        if "blob/master/build" in path:
+            raise RuntimeError(  # pragma: no cover
+                "Unexpected substring found in %r." % path)
         links.append(
             ":githublink:`GitHub|{0}|*`".format(path.replace("\\", "/").lstrip("/")))
     lines[pos] = "{0}\n\n.. only:: html\n\n    {1}\n\n".format(
@@ -554,6 +561,8 @@ def post_process_html_output(file, pdf, python, slides, exc=True,
     """
     if not os.path.exists(file):
         raise FileNotFoundError(file)  # pragma: no cover
+    if fLOG:
+        fLOG("[post_process_html_output] clean %r" % file)
     with open(file, "r", encoding="utf8") as f:
         text = f.read()
 
@@ -564,7 +573,7 @@ def post_process_html_output(file, pdf, python, slides, exc=True,
 
     # notebook replacements
     if fLOG:
-        fLOG("[post_process_html_output] ", notebook_replacements)
+        fLOG("[post_process_html_output] nb:", notebook_replacements)
     text = _notebook_replacements(text, notebook_replacements, fLOG)
 
     text = update_notebook_link(text, "html", nblinks=nblinks, fLOG=fLOG)
@@ -573,6 +582,26 @@ def post_process_html_output(file, pdf, python, slides, exc=True,
             "find:// was found in '{0}'.\nYou should add "
             "or extend 'nblinks' in conf.py.".format(file))
 
+    # js
+    if fLOG:
+        fLOG("[post_process_html_output] js: replacements")
+    repl = {'https://unpkg.com/@jupyter-widgets/html-manager@^0.20.0/dist/embed-amd.js':
+            '../_static/embed-amd.js'}
+    lines = text.split('\n')
+    new_lines = []
+    for line in lines:
+        if "https://cdnjs.cloudflare.com/ajax/libs/require.js" in line:
+            if fLOG:
+                fLOG("[post_process_html_output] js: skip %r" % line)
+            continue
+        new_lines.append(line)
+    text = "\n".join(new_lines)
+    for k, v in repl.items():
+        if k in text:
+            if fLOG:
+                fLOG("[post_process_html_output] js: replace %r -> %r" % (k, v))
+            text = text.replace(k, v)
+
     with open(file, "w", encoding="utf8") as f:
         f.write(text)
 
@@ -599,22 +628,29 @@ def post_process_slides_output(file, pdf, python, slides, exc=True,
     else:
         if not os.path.exists(file):
             raise FileNotFoundError(file)
+        if fLOG:
+            fLOG("[post_process_slides_output] clean %r" % file)
         # fold, name = os.path.split(file)
         with open(file, "r", encoding="utf8") as f:
             text = f.read()
         save = True
 
     # reveal.js
     require = "require(" in text
-    text = text.replace("reveal.js/js/reveal.js", "reveal.js/js/reveal.js")
+    text = text.replace("reveal.js/dist/reveal.css",
+                        "reveal.js/css/reveal.css")
+    text = text.replace("reveal.js/dist/theme/simple.css",
+                        "reveal.js/css/theme/simple.css")
+    text = text.replace("https://unpkg.com/@jupyter-widgets/html-manager@0.20.0/dist/embed-amd.js",
+                        "embed-amd.js")
     lines = text.split("\n")
     for i, line in enumerate(lines):
         if '<script src="reveal.js/lib/js/head.min.js"></script>' in line:
             lines[
                 i] = '<script src="reveal.js/js/jquery.min.js"></script>\n' + lines[i]
         if '<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js"></script>' in line:
             lines[i] = ""
-        if '<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js"></script>' in line:
+        if '<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/' in line:
             lines[i] = ""
         if lines[i] == "</script>" and require:
             lines[i] += '\n<script src="require.js"></script>'

src/pyquickhelper/helpgen/process_notebooks.py

@@ -16,8 +16,10 @@
 
 from .utils_sphinx_doc_helpers import HelpGenException
 from .conf_path_tools import find_latex_path, find_pandoc_path
-from .post_process import post_process_latex_output, post_process_latex_output_any, post_process_rst_output
-from .post_process import post_process_html_output, post_process_slides_output, post_process_python_output
+from .post_process import (
+    post_process_latex_output, post_process_latex_output_any,
+    post_process_rst_output, post_process_html_output,
+    post_process_slides_output, post_process_python_output)
 from .helpgen_exceptions import NotebookConvertError
 from .install_js_dep import install_javascript_tools
 from .style_css_template import THUMBNAIL_TEMPLATE, THUMBNAIL_TEMPLATE_TABLE
@@ -236,8 +238,9 @@ def _process_notebooks_in_private(fnbcexe, list_args, options_args):
                         for k, v in sorted(os.environ.items()))
         raise RuntimeError(  # pragma: no cover
             "Notebook conversion failed.\nfnbcexe\n{}\noptions_args\n{}"
-            "\nARGS:\n{}\nOUT\n{}\nERR\n{}\nENVIRON\n{}".format(
-                fnbcexe, options_args, list_args, out, err, env)) from exc
+            "\n--ARGS--\n{}\n--OUT--\n{}\n--ERR--\n{}\n--ENVIRON--\n{}"
+            "".format(fnbcexe, options_args, list_args, out, err,
+                      env)) from exc
     return out, err
 
 
@@ -527,8 +530,8 @@ def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_pat
                                 "Unable to convert a notebook\n----\n{}----\n{}\n"
                                 "---ERR---\n{}\n---OUT---\n{}".format(
                                     fnbcexe, list_args, err, out))
-                        fLOG("[_process_notebooks_in] LATEX ERR\n" + err)
-                        fLOG("[_process_notebooks_in] LATEX OUT\n" + out)
+                        fLOG("[_process_notebooks_in] LATEX --ERR--\n" + err)
+                        fLOG("[_process_notebooks_in] LATEX --OUT--\n" + out)
                     else:
                         err = err.lower()
                         if "critical" in err or "bad config" in err:
@@ -588,9 +591,9 @@ def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_pat
                         catch_exit=True, prefix_log="[latex] ", change_path=change_path)
                     if out is not None and ("Output written" in out or 'bytes written' in out):
                         # The output was produced. We ignore the return code.
-                        fLOG(
-                            "[_process_notebooks_in] WARNINGS: Latex compilation had warnings:", c)
-                        out += "\nERR\n" + err
+                        fLOG("[_process_notebooks_in] WARNINGS: "
+                             "Latex compilation had warnings:", c)
+                        out += "\n--ERR--\n" + err
                         err = ""
                     if len(err) > 0:
                         raise HelpGenException(

src/pyquickhelper/helpgen/sphinx_helper.py

@@ -57,6 +57,7 @@ def post_process_html_nb_output_static_file(build, fLOG=noLOG):
 
     res = []
     for full in explore_folder_iterfile(build, pattern=".*[.]html"):
+        modif = False
         with open(full, "r", encoding="utf8") as f:
             try:
                 content = f.read()
@@ -68,13 +69,37 @@ def post_process_html_nb_output_static_file(build, fLOG=noLOG):
                         content = content.replace(
                             "charset=cp1252", "charset=utf-8")
                     except UnicodeDecodeError:
-                        raise Exception(
-                            "unable to load: " + full + "\n" + os.path.abspath(full)) from e
+                        raise FileNotFoundError(
+                            "Unable to load %r\n%r" % (full, os.path.abspath(full))) from e
 
         if tofind in content:
             res.append(full)
-            fLOG("[post_process_html_nb_output_static_file]", full)
             content = content.replace(tofind, torep)
+            modif = True
+
+        # js
+        repl = {'https://unpkg.com/@jupyter-widgets/html-manager@^0.20.0/dist/embed-amd.js':
+                '../_static/embed-amd.js'}
+        lines = content.split('\n')
+        new_lines = []
+        for line in lines:
+            if "https://cdnjs.cloudflare.com/ajax/libs/require.js" in line:
+                if fLOG:
+                    fLOG(
+                        "[post_process_html_nb_output_static_file] js: skip %r" % line)
+                modif = True
+                continue
+            new_lines.append(line)
+        content = "\n".join(new_lines)
+        for k, v in repl.items():
+            if k in content:
+                if fLOG:
+                    fLOG("[post_process_html_output] js: replace %r -> %r" % (k, v))
+                content = content.replace(k, v)
+                modif = True
+
+        if modif:
+            fLOG("[post_process_html_nb_output_static_file] %r" % full)
             with open(full, "w", encoding="utf8") as f:
                 f.write(content)
 

src/pyquickhelper/helpgen/sphinx_main.py

@@ -255,20 +255,9 @@ def generate_help_sphinx(project_var_name, clean=False, root=".",
     *remove_unicode* can set to False or True in the documentation
     configuration file to allow or remove unicode characters
     before compiling the latex output.
-
-    .. versionchanged:: 1.7
-        Upgrade to Sphinx 1.7. It introduced a breaking
-        change with method ``app.status_iterator`` must be
-        replaced by ``status_iterator``.
-        See issue `bokeh:7520 <https://github.com/bokeh/bokeh/issues/7520>`_.
-
-    .. versionchanged:: 1.8
-        Uses own image directive.
-
-    .. versionchanged:: 1.9
-        Import ``conf.py`` in a separate process before running
-        the generation of the documentation. Do not import it
-        directly.
+    Import ``conf.py`` in a separate process before running
+    the generation of the documentation. Do not import it
+    directly.
     """
     datetime_rows = [("begin", datetime.now())]
 
@@ -966,16 +955,18 @@ def keep_line(_):  # pragma: no cover
             fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
 
         if kind == "html":
-            fLOG(
-                "##############################################################")
+            fLOG("#########################################################")
             fLOG("[generate_help_sphinx] check that index.html exists")
             findex = os.path.join(build, kind, "index.html")
             if not os.path.exists(findex):
-                raise FileNotFoundError("something went wrong, unable to find {0}\nCMD\n{1}\nOUT\n{2}\nERR\n{3}\nLAY\n{4}\nINDEX\n{5}"
-                                        .format(findex, cmd, out, err, kind, os.path.abspath(findex)))
-
-            fLOG(
-                "##############################################################")
+                raise FileNotFoundError(
+                    "something went wrong, unable to find {0}\n"
+                    "--CMD--\n{1}\n--OUT--\n{2}\n--ERR--\n{3}\n"
+                    "--LAY--\n{4}\n--INDEX--\n{5}"
+                    "".format(findex, cmd, out, err, kind,
+                              os.path.abspath(findex)))
+
+            fLOG("#########################################################")
             verification_html_format(os.path.join(build, kind), fLOG=fLOG)
 
         fLOG(