Skip to content
This repository has been archived by the owner on Jan 13, 2024. It is now read-only.

Commit

Permalink
add pdf compilation of the help
Browse files Browse the repository at this point in the history
  • Loading branch information
@sdpython
sdpython committed May 1, 2014
1 parent ffc82df commit 58bf387
Show file tree
Hide file tree
Showing 2 changed files with 183 additions and 95 deletions.
4 changes: 3 additions & 1 deletion _unittests/ut_helpgen/test_notebooks.py
View file
Expand Up @@ -30,7 +30,9 @@ def test_notebook(self) :
for file in os.listdir(temp):
os.remove(os.path.join(temp,file))

res = process_notebooks(nb, temp, temp)
p1 = r"C:\Program Files\MiKTeX 2.9\miktex\bin\x64"
p2 = r"%USERPROFILE%\AppData\Local\Pandoc"
res = process_notebooks(nb, temp, temp, latex_path = p1, pandoc_path = p2)
for _ in res:
fLOG(_)
assert os.path.exists(_)
Expand Down
274 changes: 180 additions & 94 deletions src/pyquickhelper/helpgen/sphinx_main.py
View file
@@ -1,9 +1,10 @@
"""
@file
@brief Main functions to produce the documentation for a module
@brief Contains the main function to generate the documentation
for a module designed the same way as this one, @see fn generate_help_sphinx.
"""
import os,sys, shutil, datetime, re
import os,sys, shutil, datetime, re, importlib
from pandas import DataFrame

from ..loghelper.flog import run_cmd, fLOG
Expand All @@ -29,90 +30,13 @@
"""

def get_executables_path() :
"""
returns the paths to Python, Python Scripts
@return a list of paths
"""
res = [ os.path.split(sys.executable)[0] ]
res += [ os.path.join(res[-1], "Scripts") ]
if sys.platform == "win32" :
ver = "c:\\Python%d%d" % (sys.version_info.major, sys.version_info.minor)
res += [ver ]
res += [ os.path.join(res[-1], "Scripts") ]
return res

def generate_changes_repo( chan,
source,
exception_if_empty = True,
filter_commit = lambda c : c.strip() != "documentation") :
"""
Generates a rst tables containing the changes stored by a svn or git repository,
the outcome is stored in a file.
The log comment must start with ``*`` to be taken into account.
@param chan filename to write (or None if you don't need to)
@param source source folder to get changes for
@param exception_if_empty raises an exception if empty
@param filter_commit function which accepts a commit to show on the documentation (based on the comment)
@return string (rst tables with the changes)
"""
# builds the changes files
try :
src = SourceRepository(commandline=True)
logs = src.log(path = source)
except Exception as e :
if exception_if_empty :
fLOG("error, unable to retrieve log from " + source)
raise HelpGenException("unable to retrieve log from " + source) from e
else :
logs = [ ("none", 0, datetime.datetime.now(), "-") ]
fLOG("error,",e)

if len(logs) == 0 :
fLOG("error, unable to retrieve log from " + source)
if exception_if_empty:
raise HelpGenException("retrieved logs are empty from " + source)
else :
fLOG("info, retrieved ", len(logs), " commits")

rows = [ ]
rows.append("""\n.. _l-changes:\n\n\nChanges\n=======\n\nList of recent changes:\n""")

values = []
for row in logs :
code, nbch, date, comment = row[:4]
last = row[-1]
if last.startswith("http") :
nbch = "`%s <%s>`_"% (str(nbch), last)

ds = "%04d-%02d-%02d" % (date.year, date.month, date.day)
if filter_commit(comment):
if isinstance(nbch,int) :
values.append ( ["%04d" % nbch, "%s" % ds, comment.strip("*") ] )
else :
values.append ( ["%s" % nbch, "%s" % ds, comment.strip("*") ] )

if len(values) == 0 and exception_if_empty:
raise HelpGenException("Logs were not empty but there was no comment starting with '*' from " + source + "\n" + "\n".join( [ str(_) for _ in logs ] ))

if len(values) > 0 :
tbl = DataFrame ( columns=["change number", "date", "comment"], data=values)
rows.append("\n\n" + df_to_rst(tbl, align=["1x","1x","3x"]) + "\n\n")

final = "\n".join(rows)
if chan != None :
with open(chan, "w", encoding="utf8") as f :
f.write(final)
return final

def generate_help_sphinx ( project_var_name,
clean = True,
root = ".",
filter_commit = lambda c : c.strip() != "documentation",
extra_ext = [],
nbformats = ["html", "python", "rst", "pdf"]) :
nbformats = ["html", "python", "rst", "pdf"],
layout = ["html"]) :
"""
runs the help generation
- copies every file in another folder
Expand All @@ -126,8 +50,19 @@ def generate_help_sphinx ( project_var_name,
@param filter_commit function which accepts a commit to show on the documentation (based on the comment)
@param extra_ext list of file extensions
@param nbformats requested formats for the notebooks conversion
@param layout list of formats sphinx should generate such as html, latex, pdf, docx
The result is stored in path: ``root/_doc/sphinxdoc/source``.
We assume the file ``root/_doc/sphinxdoc/source/conf.py`` exists
as well as ``root/_doc/sphinxdoc/source/index.rst``.
If you generate latex/pdf files, you should add variables ``latex_path`` and ``pandoc_path``
in your file ``conf.py`` which defines the help.
@code
latex_path = r"C:/Program Files/MiKTeX 2.9/miktex/bin/x64"
pandoc_path = r"%USERPROFILE%/AppData/Local/Pandoc"
@endcode
@example(run help generation)
@code
Expand All @@ -148,17 +83,25 @@ def generate_help_sphinx ( project_var_name,
@endcode
@endexample
"""
sys.path.append (os.path.abspath(os.path.join("_doc", "sphinxdoc","source")))
root = os.path.abspath(root)
froot = root
sys.path.append (os.path.join(root, "_doc", "sphinxdoc","source"))

src = SourceRepository(commandline=True)
version = src.version(os.path.abspath(root))
version = src.version(root)
if version != None :
with open("version.txt", "w") as f : f.write(str(version) + "\n")

# modifies the version number in conf.py
shutil.copy("README.rst", "_doc/sphinxdoc/source")
shutil.copy("LICENSE.txt", "_doc/sphinxdoc/source")

# import conf.py
theconf = importlib.import_module('conf')
if theconf is None:
raise ImportError("unable to import conf.py which defines the help generation")
latex_path = theconf.__dict__.get("latex_path",r"C:\Program Files\MiKTeX 2.9\miktex\bin\x64")
pandoc_path = theconf.__dict__.get("pandoc_path",r"%USERPROFILE%\AppData\Local\Pandoc")

#changes
chan = os.path.join (root, "_doc", "sphinxdoc", "source", "filechanges.rst")
Expand Down Expand Up @@ -197,7 +140,9 @@ def generate_help_sphinx ( project_var_name,
nbs = process_notebooks(notebooks,
build=build,
outfold=notebook_doc,
formats=nbformats)
formats=nbformats,
latex_path=latex_path,
pandoc_path=pandoc_path)
add_notebook_page(nbs, os.path.join(notebook_doc,"..","all_notebooks.rst"))

# run the documentation generation
Expand Down Expand Up @@ -239,22 +184,108 @@ def generate_help_sphinx ( project_var_name,
cmd = "make.bat clean".split ()
run_cmd (cmd, wait = True)

cmd = "make.bat html".split ()

# This instruction should work but it does not. Sphinx seems to be stuck.
#run_cmd (cmd, wait = True, secure="make_help.log", stop_waiting_if = lambda v : "build succeeded" in v)
# The following one works but opens a extra windows.
os.system("make html")
for lay in layout :
if lay == "pdf":
lay = "latex"
cmd = "make {0}".format(lay)
# This instruction should work but it does not. Sphinx seems to be stuck.
#run_cmd (cmd, wait = True, secure="make_help.log", stop_waiting_if = lambda v : "build succeeded" in v)
# The following one works but opens a extra windows.
os.system(cmd)
if lay == "latex":
post_process_latex_output(froot)

if "pdf" in layout:
compile_latex_output(froot, latex_path)

# end
os.chdir (pa)

def get_executables_path() :
"""
returns the paths to Python, Python Scripts
@return a list of paths
"""
res = [ os.path.split(sys.executable)[0] ]
res += [ os.path.join(res[-1], "Scripts") ]
if sys.platform == "win32" :
ver = "c:\\Python%d%d" % (sys.version_info.major, sys.version_info.minor)
res += [ver ]
res += [ os.path.join(res[-1], "Scripts") ]
return res

def generate_changes_repo( chan,
source,
exception_if_empty = True,
filter_commit = lambda c : c.strip() != "documentation") :
"""
Generates a rst tables containing the changes stored by a svn or git repository,
the outcome is stored in a file.
The log comment must start with ``*`` to be taken into account.
@param chan filename to write (or None if you don't need to)
@param source source folder to get changes for
@param exception_if_empty raises an exception if empty
@param filter_commit function which accepts a commit to show on the documentation (based on the comment)
@return string (rst tables with the changes)
"""
# builds the changes files
try :
src = SourceRepository(commandline=True)
logs = src.log(path = source)
except Exception as e :
if exception_if_empty :
fLOG("error, unable to retrieve log from " + source)
raise HelpGenException("unable to retrieve log from " + source) from e
else :
logs = [ ("none", 0, datetime.datetime.now(), "-") ]
fLOG("error,",e)

if len(logs) == 0 :
fLOG("error, unable to retrieve log from " + source)
if exception_if_empty:
raise HelpGenException("retrieved logs are empty from " + source)
else :
fLOG("info, retrieved ", len(logs), " commits")

rows = [ ]
rows.append("""\n.. _l-changes:\n\n\nChanges\n=======\n\nList of recent changes:\n""")

values = []
for row in logs :
code, nbch, date, comment = row[:4]
last = row[-1]
if last.startswith("http") :
nbch = "`%s <%s>`_"% (str(nbch), last)

ds = "%04d-%02d-%02d" % (date.year, date.month, date.day)
if filter_commit(comment):
if isinstance(nbch,int) :
values.append ( ["%04d" % nbch, "%s" % ds, comment.strip("*") ] )
else :
values.append ( ["%s" % nbch, "%s" % ds, comment.strip("*") ] )

if len(values) == 0 and exception_if_empty:
raise HelpGenException("Logs were not empty but there was no comment starting with '*' from " + source + "\n" + "\n".join( [ str(_) for _ in logs ] ))

if len(values) > 0 :
tbl = DataFrame ( columns=["change number", "date", "comment"], data=values)
rows.append("\n\n" + df_to_rst(tbl, align=["1x","1x","3x"]) + "\n\n")

final = "\n".join(rows)
if chan != None :
with open(chan, "w", encoding="utf8") as f :
f.write(final)
return final

def process_notebooks( notebooks,
outfold,
build,
pandoc_path = "%USERPROFILE%\\AppData\\Local\\Pandoc",
formats = ["html", "python", "rst", "pdf"],
latex_path = r"C:\Program Files\MiKTeX 2.9\miktex\bin\x64"):
latex_path,
pandoc_path,
formats = ["html", "python", "rst", "pdf"]
):
"""
converts notebooks into html, rst, latex using
`nbconvert <http://ipython.org/ipython-doc/rel-1.0.0/interactive/nbconvert.html>`_.
Expand Down Expand Up @@ -547,4 +578,59 @@ def add_notebook_page(nbs, fileout):
f.write("\n".join(rows))
return fileout


def post_process_latex_output(root):
"""
post process the latex file produced by sphinx
@param root root path
"""
build = os.path.join(root, "_doc", "sphinxdoc","build","latex")
for tex in os.listdir(build):
if tex.endswith(".tex"):
file = os.path.join(build,tex)
fLOG("modify file", file)
with open(file, "r", encoding="utf8") as f : content = f.read()
content = post_process_latex(content)
with open(file, "w", encoding="utf8") as f : f.write(content)

def post_process_latex(st):
"""
modifies a latex file after its generation by sphinx
@param st string
@return string
"""
st = st.replace("<br />","\\\\")
st = st.replace("\\maketitle","\\maketitle\n\n\\newchapter{Introduction}")

# first section
lines = st.split("\n")
for i,line in enumerate(lines):
if "\\section" in line :
lines[i] = "\\newchapter{Documentation}\n" + lines[i]
break

st = st.replace("\\chapter", "\\section")
st = st.replace("\\newchapter", "\\chapter")
return st

def compile_latex_output(root, latex_path):
"""
compiles the latex documents
@param root root
@param latex_path path to the compilter
"""
lat = os.path.join(latex_path, "pdflatex.exe")
build = os.path.join(root, "_doc", "sphinxdoc","build","latex")
for tex in os.listdir(build):
if tex.endswith(".tex"):
file = os.path.join(build, tex)
c = '"{0}" "{1}" -output-directory="{2}"'.format(lat, file, build)
out,err = run_cmd(c,wait=True, do_not_log = False, log_error=False)
if len(err) > 0 :
raise HelpGenException(err)
# second compilation
out,err = run_cmd(c,wait=True, do_not_log = False, log_error=False)
if len(err) > 0 :
raise HelpGenException(err)

0 comments on commit 58bf387

Please sign in to comment.