diff --git a/_unittests/ut_helpgen/test_LONG_full_documentation_module_template.py b/_unittests/ut_helpgen/test_LONG_full_documentation_module_template.py
index e4a09a01d..baf34ae89 100644
--- a/_unittests/ut_helpgen/test_LONG_full_documentation_module_template.py
+++ b/_unittests/ut_helpgen/test_LONG_full_documentation_module_template.py
@@ -141,7 +141,7 @@ def __len__(self):
fLOG("[test_full_documentation] **********************************")
direct_call = i % 2 == 0
- layout = ["pdf", "html"]
+ layout = ["html"]
logger1 = getLogger("docassert")
logger2 = getLogger("tocdelay")
@@ -296,6 +296,23 @@ def __len__(self):
raise Exception(
"Unable to find '{0}' in\n{1}".format(tofind, content))
+ # notebook links
+ files = [os.path.join(root, "_doc", "sphinxdoc", "build", "html",
+ "notebooks", "custom_notebooks.html"),
+ os.path.join(root, "_doc", "sphinxdoc", "build", "html",
+ "notebooks", "custom_notebooks.slides.html")
+ ]
+ for name in files:
+ with open(name, "r", encoding="utf-8") as f:
+ content = f.read()
+ if "https://unpkg.com/@jupyter-widgets/html-manager@%5E0.20.0/dist/embed-amd.js" in content:
+ raise AssertionError("Absolute link in %r." % name)
+ if "https://cdnjs.cloudflare" in content:
+ raise AssertionError(
+ "Absolute cloudflare link in %r." % name)
+ if "reveal.js/dist" in content:
+ raise AssertionError("Wrong link in slides %r." % name)
+
# final check
logs = os.path.join(temp, "log_custom_000.txt")
with open(logs, "r", encoding='utf-8') as f:
diff --git a/_unittests/ut_helpgen/test_LONG_full_documentation_module_template_rst.py b/_unittests/ut_helpgen/test_LONG_full_documentation_module_template_rst.py
index a22484233..63a0cf766 100644
--- a/_unittests/ut_helpgen/test_LONG_full_documentation_module_template_rst.py
+++ b/_unittests/ut_helpgen/test_LONG_full_documentation_module_template_rst.py
@@ -24,14 +24,15 @@ def test_full_documentation_module_template_rst(self):
"""
This test might fail in sphinx-gallery due to a very long filename.
Please look into the following commit:
- https://github.com/sdpython/sphinx-gallery/commit/3ae9f13250cf25c75e1b17b2fade98b7a9940b0d.
+ https://github.com/sdpython/sphinx-gallery/commit/
+ 3ae9f13250cf25c75e1b17b2fade98b7a9940b0d.
"""
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
- if is_travis_or_appveyor() in ('travis', 'appveyor') or sys.version_info[0] == 2:
+ if is_travis_or_appveyor() in ('travis', 'appveyor'):
# travis fails due to the following:
# sitep = [_ for _ in site.getsitepackages() if "packages" in _]
# AttributeError: 'module' object has no attribute
diff --git a/src/pyquickhelper/filehelper/compression_helper.py b/src/pyquickhelper/filehelper/compression_helper.py
index 07ac9331c..ec70adc11 100644
--- a/src/pyquickhelper/filehelper/compression_helper.py
+++ b/src/pyquickhelper/filehelper/compression_helper.py
@@ -298,7 +298,9 @@ def zip7_files(filename_7z, file_set, fLOG=noLOG, temp_folder="."):
out, err = run_cmd(cmd, wait=True)
if "Error:" in out or not os.path.exists(filename_7z):
raise FileException( # pragma: no cover
- "An error occurred with cmd: '{0}'\nOUT:\n{1}\nERR\n{2}\n----".format(cmd, out, err))
+ "An error occurred with cmd: '{0}'\n"
+ "--OUT--\n{1}\n--ERR--\n{2}\n----".format(
+ cmd, out, err))
return len(file_set)
@@ -462,7 +464,9 @@ def unrar_files(zipf, where_to=None, fLOG=noLOG, fvalid=None, remove_space=True)
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
if len(err) > 0 or "Error:" in out:
raise FileException(
- "Unable to unrar file '{0}'\nOUT\n{1}\nERR\n{2}".format(zipf, out, err))
+ "Unable to unrar file '{0}'\n"
+ "--OUT--\n{1}\n--ERR--\n{2}".format(
+ zipf, out, err))
return explore_folder(where_to)[1]
else:
diff --git a/src/pyquickhelper/helpgen/install_custom.py b/src/pyquickhelper/helpgen/install_custom.py
index dbcd50453..19bee1ebf 100644
--- a/src/pyquickhelper/helpgen/install_custom.py
+++ b/src/pyquickhelper/helpgen/install_custom.py
@@ -63,10 +63,9 @@ def download_revealjs(temp_folder=".", unzip_to=".", fLOG=print,
for r in res:
if os.path.isdir(r):
continue
- if ".gitignore" in r or ".travis.yml" in r or "index.html" in r \
- or ".appveyor.yml" in r \
- or "requirement" in r \
- or "README" in r or "CONTRIBUTING.md" in r:
+ if (".gitignore" in r or ".travis.yml" in r or "index.html" in r or
+ ".appveyor.yml" in r or "requirement" in r or "README" in r or
+ "CONTRIBUTING.md" in r):
os.remove(r)
elif "/test/" in r.replace("\\", "/"):
os.remove(r)
diff --git a/src/pyquickhelper/helpgen/install_js_dep.py b/src/pyquickhelper/helpgen/install_js_dep.py
index 6b6734396..d1587e1a7 100644
--- a/src/pyquickhelper/helpgen/install_js_dep.py
+++ b/src/pyquickhelper/helpgen/install_js_dep.py
@@ -6,7 +6,8 @@
import os
from ..loghelper.flog import noLOG
from .install_custom import download_revealjs, download_requirejs
-from ..filehelper import synchronize_folder, change_file_status
+from ..filehelper import (
+ synchronize_folder, change_file_status, download)
def install_javascript_tools(root, dest, fLOG=noLOG,
@@ -34,7 +35,6 @@ def install_javascript_tools(root, dest, fLOG=noLOG,
else:
rev = os.path.join(dest, "reveal.js")
if not os.path.exists(rev):
-
folder = os.path.dirname(revealjs.__file__)
js = os.path.join(folder, "templates", "revealjs", "static")
os.mkdir(rev)
@@ -52,4 +52,13 @@ def install_javascript_tools(root, dest, fLOG=noLOG,
else:
one = [expected]
lfiles.extend(one)
+
+ # embed-ams.js
+ expected = os.path.join(dest, "embed-amd.js")
+ if not os.path.exists(expected):
+ url = "https://unpkg.com/@jupyter-widgets/html-manager@0.20.0/dist/embed-amd.js"
+ one = [download(url, dest, fLOG=fLOG)]
+ else:
+ one = [expected]
+ lfiles.extend(one)
return lfiles
diff --git a/src/pyquickhelper/helpgen/pandoc_helper.py b/src/pyquickhelper/helpgen/pandoc_helper.py
index 31f85a2e1..3fc1a5e58 100644
--- a/src/pyquickhelper/helpgen/pandoc_helper.py
+++ b/src/pyquickhelper/helpgen/pandoc_helper.py
@@ -19,8 +19,10 @@ def call_pandoc(params, fLOG=noLOG):
cmd = '"{0}" {1}'.format(pandoc, params)
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
if err is not None and "Cannot decode byte" in err:
- raise Exception(
- "Issue with pandac:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
+ raise RuntimeError(
+ "Issue with pandoc:\n{0}\n"
+ "--OUT--\n{1}\n--ERR--\n{2}".format(
+ cmd, out, err))
return out, err
diff --git a/src/pyquickhelper/helpgen/post_process.py b/src/pyquickhelper/helpgen/post_process.py
index f7f5bd5bb..b483b57fe 100644
--- a/src/pyquickhelper/helpgen/post_process.py
+++ b/src/pyquickhelper/helpgen/post_process.py
@@ -179,6 +179,8 @@ def post_process_latex_output(root, doall, latex_book=False, exc=True,
"""
if os.path.isfile(root):
file = root
+ if fLOG:
+ fLOG("[post_process_latex_output] clean %r" % file)
with open(file, "r", encoding="utf8") as f:
content = f.read()
with open(file + ".tex1~", "w", encoding="utf8") as f:
@@ -224,6 +226,8 @@ def post_process_python_output(root, doall, exc=True, nblinks=None, fLOG=None, n
"""
if os.path.isfile(root):
file = root
+ if fLOG:
+ fLOG("[post_process_python_output] clean %r" % file)
with open(file, "r", encoding="utf8") as f:
content = f.read()
content = post_process_python(
@@ -301,7 +305,7 @@ def post_process_rst_output(file, html, pdf, python, slides, is_notebook=False,
and checks that audio is only included in :epkg:`HTML`.
"""
if fLOG:
- fLOG("[post_process_rst_output] %r" % file)
+ fLOG("[post_process_rst_output] clean %r" % file)
name = os.path.split(file)[1]
noext = os.path.splitext(name)[0]
@@ -455,6 +459,9 @@ def startss(line):
docname = selected[0]
path = docname[len(folder):]
break
+ if "blob/master/build" in path:
+ raise RuntimeError( # pragma: no cover
+ "Unexpected substring found in %r." % path)
links.append(
":githublink:`GitHub|{0}|*`".format(path.replace("\\", "/").lstrip("/")))
lines[pos] = "{0}\n\n.. only:: html\n\n {1}\n\n".format(
@@ -554,6 +561,8 @@ def post_process_html_output(file, pdf, python, slides, exc=True,
"""
if not os.path.exists(file):
raise FileNotFoundError(file) # pragma: no cover
+ if fLOG:
+ fLOG("[post_process_html_output] clean %r" % file)
with open(file, "r", encoding="utf8") as f:
text = f.read()
@@ -564,7 +573,7 @@ def post_process_html_output(file, pdf, python, slides, exc=True,
# notebook replacements
if fLOG:
- fLOG("[post_process_html_output] ", notebook_replacements)
+ fLOG("[post_process_html_output] nb:", notebook_replacements)
text = _notebook_replacements(text, notebook_replacements, fLOG)
text = update_notebook_link(text, "html", nblinks=nblinks, fLOG=fLOG)
@@ -573,6 +582,26 @@ def post_process_html_output(file, pdf, python, slides, exc=True,
"find:// was found in '{0}'.\nYou should add "
"or extend 'nblinks' in conf.py.".format(file))
+ # js
+ if fLOG:
+ fLOG("[post_process_html_output] js: replacements")
+ repl = {'https://unpkg.com/@jupyter-widgets/html-manager@^0.20.0/dist/embed-amd.js':
+ '../_static/embed-amd.js'}
+ lines = text.split('\n')
+ new_lines = []
+ for line in lines:
+ if "https://cdnjs.cloudflare.com/ajax/libs/require.js" in line:
+ if fLOG:
+ fLOG("[post_process_html_output] js: skip %r" % line)
+ continue
+ new_lines.append(line)
+ text = "\n".join(new_lines)
+ for k, v in repl.items():
+ if k in text:
+ if fLOG:
+ fLOG("[post_process_html_output] js: replace %r -> %r" % (k, v))
+ text = text.replace(k, v)
+
with open(file, "w", encoding="utf8") as f:
f.write(text)
@@ -599,6 +628,8 @@ def post_process_slides_output(file, pdf, python, slides, exc=True,
else:
if not os.path.exists(file):
raise FileNotFoundError(file)
+ if fLOG:
+ fLOG("[post_process_slides_output] clean %r" % file)
# fold, name = os.path.split(file)
with open(file, "r", encoding="utf8") as f:
text = f.read()
@@ -606,7 +637,12 @@ def post_process_slides_output(file, pdf, python, slides, exc=True,
# reveal.js
require = "require(" in text
- text = text.replace("reveal.js/js/reveal.js", "reveal.js/js/reveal.js")
+ text = text.replace("reveal.js/dist/reveal.css",
+ "reveal.js/css/reveal.css")
+ text = text.replace("reveal.js/dist/theme/simple.css",
+ "reveal.js/css/theme/simple.css")
+ text = text.replace("https://unpkg.com/@jupyter-widgets/html-manager@0.20.0/dist/embed-amd.js",
+ "embed-amd.js")
lines = text.split("\n")
for i, line in enumerate(lines):
if '' in line:
@@ -614,7 +650,7 @@ def post_process_slides_output(file, pdf, python, slides, exc=True,
i] = '\n' + lines[i]
if '' in line:
lines[i] = ""
- if '' in line:
+ if ''
diff --git a/src/pyquickhelper/helpgen/process_notebooks.py b/src/pyquickhelper/helpgen/process_notebooks.py
index 4c5f559dd..01b761051 100644
--- a/src/pyquickhelper/helpgen/process_notebooks.py
+++ b/src/pyquickhelper/helpgen/process_notebooks.py
@@ -16,8 +16,10 @@
from .utils_sphinx_doc_helpers import HelpGenException
from .conf_path_tools import find_latex_path, find_pandoc_path
-from .post_process import post_process_latex_output, post_process_latex_output_any, post_process_rst_output
-from .post_process import post_process_html_output, post_process_slides_output, post_process_python_output
+from .post_process import (
+ post_process_latex_output, post_process_latex_output_any,
+ post_process_rst_output, post_process_html_output,
+ post_process_slides_output, post_process_python_output)
from .helpgen_exceptions import NotebookConvertError
from .install_js_dep import install_javascript_tools
from .style_css_template import THUMBNAIL_TEMPLATE, THUMBNAIL_TEMPLATE_TABLE
@@ -236,8 +238,9 @@ def _process_notebooks_in_private(fnbcexe, list_args, options_args):
for k, v in sorted(os.environ.items()))
raise RuntimeError( # pragma: no cover
"Notebook conversion failed.\nfnbcexe\n{}\noptions_args\n{}"
- "\nARGS:\n{}\nOUT\n{}\nERR\n{}\nENVIRON\n{}".format(
- fnbcexe, options_args, list_args, out, err, env)) from exc
+ "\n--ARGS--\n{}\n--OUT--\n{}\n--ERR--\n{}\n--ENVIRON--\n{}"
+ "".format(fnbcexe, options_args, list_args, out, err,
+ env)) from exc
return out, err
@@ -527,8 +530,8 @@ def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_pat
"Unable to convert a notebook\n----\n{}----\n{}\n"
"---ERR---\n{}\n---OUT---\n{}".format(
fnbcexe, list_args, err, out))
- fLOG("[_process_notebooks_in] LATEX ERR\n" + err)
- fLOG("[_process_notebooks_in] LATEX OUT\n" + out)
+ fLOG("[_process_notebooks_in] LATEX --ERR--\n" + err)
+ fLOG("[_process_notebooks_in] LATEX --OUT--\n" + out)
else:
err = err.lower()
if "critical" in err or "bad config" in err:
@@ -588,9 +591,9 @@ def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_pat
catch_exit=True, prefix_log="[latex] ", change_path=change_path)
if out is not None and ("Output written" in out or 'bytes written' in out):
# The output was produced. We ignore the return code.
- fLOG(
- "[_process_notebooks_in] WARNINGS: Latex compilation had warnings:", c)
- out += "\nERR\n" + err
+ fLOG("[_process_notebooks_in] WARNINGS: "
+ "Latex compilation had warnings:", c)
+ out += "\n--ERR--\n" + err
err = ""
if len(err) > 0:
raise HelpGenException(
diff --git a/src/pyquickhelper/helpgen/sphinx_helper.py b/src/pyquickhelper/helpgen/sphinx_helper.py
index 49dc4518d..384860bd2 100644
--- a/src/pyquickhelper/helpgen/sphinx_helper.py
+++ b/src/pyquickhelper/helpgen/sphinx_helper.py
@@ -57,6 +57,7 @@ def post_process_html_nb_output_static_file(build, fLOG=noLOG):
res = []
for full in explore_folder_iterfile(build, pattern=".*[.]html"):
+ modif = False
with open(full, "r", encoding="utf8") as f:
try:
content = f.read()
@@ -68,13 +69,37 @@ def post_process_html_nb_output_static_file(build, fLOG=noLOG):
content = content.replace(
"charset=cp1252", "charset=utf-8")
except UnicodeDecodeError:
- raise Exception(
- "unable to load: " + full + "\n" + os.path.abspath(full)) from e
+ raise FileNotFoundError(
+ "Unable to load %r\n%r" % (full, os.path.abspath(full))) from e
if tofind in content:
res.append(full)
- fLOG("[post_process_html_nb_output_static_file]", full)
content = content.replace(tofind, torep)
+ modif = True
+
+ # js
+ repl = {'https://unpkg.com/@jupyter-widgets/html-manager@^0.20.0/dist/embed-amd.js':
+ '../_static/embed-amd.js'}
+ lines = content.split('\n')
+ new_lines = []
+ for line in lines:
+ if "https://cdnjs.cloudflare.com/ajax/libs/require.js" in line:
+ if fLOG:
+ fLOG(
+ "[post_process_html_nb_output_static_file] js: skip %r" % line)
+ modif = True
+ continue
+ new_lines.append(line)
+ content = "\n".join(new_lines)
+ for k, v in repl.items():
+ if k in content:
+ if fLOG:
+ fLOG("[post_process_html_output] js: replace %r -> %r" % (k, v))
+ content = content.replace(k, v)
+ modif = True
+
+ if modif:
+ fLOG("[post_process_html_nb_output_static_file] %r" % full)
with open(full, "w", encoding="utf8") as f:
f.write(content)
diff --git a/src/pyquickhelper/helpgen/sphinx_main.py b/src/pyquickhelper/helpgen/sphinx_main.py
index 6b6d88659..723da9f44 100644
--- a/src/pyquickhelper/helpgen/sphinx_main.py
+++ b/src/pyquickhelper/helpgen/sphinx_main.py
@@ -255,20 +255,9 @@ def generate_help_sphinx(project_var_name, clean=False, root=".",
*remove_unicode* can set to False or True in the documentation
configuration file to allow or remove unicode characters
before compiling the latex output.
-
- .. versionchanged:: 1.7
- Upgrade to Sphinx 1.7. It introduced a breaking
- change with method ``app.status_iterator`` must be
- replaced by ``status_iterator``.
- See issue `bokeh:7520 `_.
-
- .. versionchanged:: 1.8
- Uses own image directive.
-
- .. versionchanged:: 1.9
- Import ``conf.py`` in a separate process before running
- the generation of the documentation. Do not import it
- directly.
+ Import ``conf.py`` in a separate process before running
+ the generation of the documentation. Do not import it
+ directly.
"""
datetime_rows = [("begin", datetime.now())]
@@ -966,16 +955,18 @@ def keep_line(_): # pragma: no cover
fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
if kind == "html":
- fLOG(
- "##############################################################")
+ fLOG("#########################################################")
fLOG("[generate_help_sphinx] check that index.html exists")
findex = os.path.join(build, kind, "index.html")
if not os.path.exists(findex):
- raise FileNotFoundError("something went wrong, unable to find {0}\nCMD\n{1}\nOUT\n{2}\nERR\n{3}\nLAY\n{4}\nINDEX\n{5}"
- .format(findex, cmd, out, err, kind, os.path.abspath(findex)))
-
- fLOG(
- "##############################################################")
+ raise FileNotFoundError(
+ "something went wrong, unable to find {0}\n"
+ "--CMD--\n{1}\n--OUT--\n{2}\n--ERR--\n{3}\n"
+ "--LAY--\n{4}\n--INDEX--\n{5}"
+ "".format(findex, cmd, out, err, kind,
+ os.path.abspath(findex)))
+
+ fLOG("#########################################################")
verification_html_format(os.path.join(build, kind), fLOG=fLOG)
fLOG(
diff --git a/src/pyquickhelper/loghelper/pwd_helper.py b/src/pyquickhelper/loghelper/pwd_helper.py
index ce56ef285..ffb2e1a85 100644
--- a/src/pyquickhelper/loghelper/pwd_helper.py
+++ b/src/pyquickhelper/loghelper/pwd_helper.py
@@ -35,7 +35,7 @@ def set_password(system, username, password, lib='keyrings.cryptfile',
system, username))
return
if lib == 'keyrings.cryptfile':
- from keyrings.cryptfile.cryptfile import CryptFileKeyring
+ from keyrings.cryptfile.cryptfile import CryptFileKeyring # pylint: disable=E0401
kr = CryptFileKeyring()
kr.keyring_key = getenv("KEYRING_CRYPTFILE_PASSWORD")
if kr.keyring_key is None and ask:
@@ -71,7 +71,7 @@ def get_password(system, username, lib='keyrings.cryptfile',
system, username))
return pwd
if lib == 'keyrings.cryptfile':
- from keyrings.cryptfile.cryptfile import CryptFileKeyring
+ from keyrings.cryptfile.cryptfile import CryptFileKeyring # pylint: disable=E0401
kr = CryptFileKeyring()
kr.keyring_key = getenv("KEYRING_CRYPTFILE_PASSWORD")
if kr.keyring_key is None and ask:
diff --git a/src/pyquickhelper/pycode/code_helper.py b/src/pyquickhelper/pycode/code_helper.py
index dc9891452..653940ebc 100644
--- a/src/pyquickhelper/pycode/code_helper.py
+++ b/src/pyquickhelper/pycode/code_helper.py
@@ -95,8 +95,10 @@ def cdiff(lines):
"\n".join(lines2), options=autopep8.parse_args(options))
if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2):
- raise ValueError("Resulting file is empty for '{3}',\ninitial number of lines {0} encoding='{1}' diff={2}".format(
- len(lines), encoding, diff, filename))
+ raise ValueError(
+ "Resulting file is empty for '{3}',\ninitial number of lines "
+ "{0} encoding='{1}' diff={2}".format(
+ len(lines), encoding, diff, filename))
if filename is None:
return r
elif r != initial_content:
diff --git a/src/pyquickhelper/sphinxext/sphinx_runpython_extension.py b/src/pyquickhelper/sphinxext/sphinx_runpython_extension.py
index 2e76a6bd1..52e24d92c 100644
--- a/src/pyquickhelper/sphinxext/sphinx_runpython_extension.py
+++ b/src/pyquickhelper/sphinxext/sphinx_runpython_extension.py
@@ -179,8 +179,10 @@ def interpret(s):
return out, err, None
except Exception as ee:
if not exception:
- message = "SCRIPT:\n{0}\nPARAMS\n{1}\nCOMMENT\n{2}\nERR\n{3}\nOUT\n{4}\nEXC\n{5}".format(
- script, params, comment, "", str(ee), ee)
+ message = ("--SCRIPT--\n{0}\n--PARAMS--\n{1}\n--COMMENT--\n"
+ "{2}\n--ERR--\n{3}\n--OUT--\n{4}\n--EXC--\n{5}"
+ "").format(script, params, comment, "",
+ str(ee), ee)
if exc_path:
message += "\n---EXC--\n{0}".format(exc_path)
raise RunPythonExecutionError(message) from ee
@@ -243,8 +245,11 @@ def interpret(s):
_ for _ in lines if "sphinx_runpython_extension.py" not in _)
if not exception:
- message = "SCRIPT:\n{0}\nPARAMS\n{1}\nCOMMENT\n{2}\nERR\n{3}\nOUT\n{4}\nEXC\n{5}\nTRACEBACK\n{6}".format(
- script, params, comment, gout, gerr, ee, excs)
+ message = ("--SCRIPT--\n{0}\n--PARAMS--\n{1}\n--COMMENT--"
+ "\n{2}\n--ERR--\n{3}\n--OUT--\n{4}\n--EXC--"
+ "\n{5}\n--TRACEBACK--\n{6}").format(
+ script, params, comment, gout, gerr,
+ ee, excs)
raise RunPythonExecutionError(message) from ee
return (gout + "\n" + gerr), (gerr + "\n" + excs), None