Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

PGF backend, fix #1116, #1118 and #1128 #1124

Merged
merged 6 commits into from

2 participants

@pwuertz
Collaborator

This PR contains the fixes for #1116, #1118 and #1128.

@mdboom
Owner

On a machine that still has the "too old" TeX, I now get this, which is correct -- they are known failures (and I've turned off the KnownFailure class here just so I can see exactly where they are coming from).

>nosetests-2.7 matplotlib.tests.test_backend_pgf
/home/mdboom/python/lib/python2.7/site-packages/matplotlib/__init__.py:768: UserWarning: Bad val "True" on line #12
        "text.hinting: True
"
        in file "/home/mdboom/.matplotlib/matplotlibrc"
        hinting should be 'auto', 'native', 'either' or 'none'
  "%s"\n\t%s' % (val, cnt, line, fname, msg))
EEE
======================================================================
ERROR: matplotlib.tests.test_backend_pgf.test_xelatex
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/mdboom/python/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/testing/decorators.py", line 47, in failer
    raise KnownFailureTest(msg) # An error here when running nose means that you don't have the matplotlib.testing.noseclasses:KnownFailure plugin in use.
KnownFailureTest: xelatex + pgf is required

======================================================================
ERROR: matplotlib.tests.test_backend_pgf.test_pdflatex
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/mdboom/python/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/testing/decorators.py", line 51, in failer
    raise KnownFailureDidNotFailTest(msg)
KnownFailureDidNotFailTest: pdflatex + pgf is required

======================================================================
ERROR: matplotlib.tests.test_backend_pgf.test_rcupdate
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/mdboom/python/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/testing/decorators.py", line 47, in failer
    raise KnownFailureTest(msg) # An error here when running nose means that you don't have the matplotlib.testing.noseclasses:KnownFailure plugin in use.
KnownFailureTest: xelatex and pdflatex + pgf required

----------------------------------------------------------------------
Ran 3 tests in 5.867s

However, on a machine where I upgraded the TeX to TeXLive 2012, I'm getting this:


.FF
======================================================================
FAIL: matplotlib.tests.test_backend_pgf.test_pdflatex
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/mdboom/python/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/testing/decorators.py", line 39, in failer
    result = f(*args, **kwargs)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/tests/test_backend_pgf.py", line 39, in backend_switcher
    result = func(*args, **kwargs)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/tests/test_backend_pgf.py", line 91, in test_pdflatex
    compare_figure('pgf_pdflatex.pdf')
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/tests/test_backend_pgf.py", line 55, in compare_figure
    raise ImageComparisonFailure('images not close: %s vs. %s' % (actual, expected))
ImageComparisonFailure: images not close: /home/mdboom/Work/tmp/matplotlib/tests/result_images/test_backend_pgf/pgf_pdflatex.pdf vs. /home/mdboom/Work/tmp/matplotlib/tests/result_images/test_backend_
pgf/expected_pgf_pdflatex.pdf

======================================================================
FAIL: matplotlib.tests.test_backend_pgf.test_rcupdate
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/mdboom/python/lib/python2.7/site-packages/nose/case.py", line 197, in runTest
    self.test(*self.arg)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/testing/decorators.py", line 39, in failer
    result = f(*args, **kwargs)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/tests/test_backend_pgf.py", line 39, in backend_switcher
    result = func(*args, **kwargs)
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/tests/test_backend_pgf.py", line 119, in test_rcupdate
    compare_figure('pgf_rcupdate%d.pdf' % (i+1))
  File "/home/mdboom/python/lib/python2.7/site-packages/matplotlib/tests/test_backend_pgf.py", line 55, in compare_figure
    raise ImageComparisonFailure('images not close: %s vs. %s' % (actual, expected))
ImageComparisonFailure: images not close: /home/mdboom/Work/tmp/matplotlib/tests/result_images/test_backend_pgf/pgf_rcupdate2.pdf vs. /home/mdboom/Work/tmp/matplotlib/tests/result_images/test_backend
_pgf/expected_pgf_rcupdate2.pdf

----------------------------------------------------------------------
Ran 3 tests in 4.909s

I've put my PDF files up here:

http://mdboom.github.com/scraps/pgf_pdflatex.pdf

http://mdboom.github.com/scraps/pgf_rcparams2.pdf

As for the Python 3 stuff, I plan to tackle #983 today, and that should hopefully get the python 3 tests running for you.

@pwuertz
Collaborator

Downloading pgf_rcparams2.pdf doesn't work for me, but from the pgf_pdflatex.pdf you uploaded I get that the default fonts of our latex environments are slightly different. The difference is very minimal and shouldn't cause the test to fail. When I wrote the tests I lowered the default tolerance of 1e-3 to 1e-4 because the test didn't recognize when I changed the fonts from serif to sans. Seems that tol=1e-4 is too restrictive. I reverted the tolerance back to 1e-3. Does this work for you?

@pwuertz
Collaborator

Oh.. and that knownfailureif-decorator confuses me.. for some reason, the pdflatex test actually worked with your old tex distribution.. and the test raised a KnownFailureDidNotFailTest exception. What I'm actually looking for is a decorator that doesn't try to verify if the test really fails and blames me if it succeeds :).

@mdboom
Owner

If I change the tolerance to 5e-3, it works for me.

@pwuertz
Collaborator

So tol=1e-3 didn't work for you neither? I'm using 5e-3 now. Furthermore, I discovered SkipTest in nose.plugins.skip which does exactly what I wanted. It doesn't require that a test that is marked as known-to-fail actually fails.

@pwuertz
Collaborator

Ahh ok.. I now see whats the big difference between those images.. Its just ghostscript being horrible at rendering text for some (many) fonts. I checked the PDFs using the gnome viewer and gimp (both cairo based) and saw no difference. The gs converted PNG files both look horrible but in a different way.

@mdboom
Owner

I first tried a tol of 1e-3, but that was still too strict. 5e-3 was the lowest I could go to get the tests to pass.

That's a bummer about gs -- we could probably move to another renderer down the road, but not for this release. I think we just raise the tolerance for now.

@pwuertz
Collaborator

I finished running the tests using python3. Some additional ResourceWarning messages are emitted when using py3. I got rid of the ones being traced back to backend_pgf.py. These remain:

/usr/lib/python3.2/subprocess.py:650: ResourceWarning: unclosed file <_io.FileIO name=11 mode='rb'>

Nothing to be concerned about though. I think these fixes are good to go.

@mdboom
Owner

Thanks for working through this so quickly!

@mdboom mdboom merged commit c981774 into from
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
View
208 lib/matplotlib/backends/backend_pgf.py
@@ -18,6 +18,7 @@
from matplotlib import _png, rcParams
from matplotlib import font_manager
from matplotlib.ft2font import FT2Font
+from matplotlib.cbook import is_string_like, is_writable_file_like
###############################################################################
@@ -220,15 +221,33 @@ def _build_latex_header():
# Create LaTeX header with some content, else LaTeX will load some
# math fonts later when we don't expect the additional output on stdout.
# TODO: is this sufficient?
- latex_header = u"""\\documentclass{minimal}
-%s
-%s
-\\begin{document}
-text $math \mu$ %% force latex to load fonts now
-\\typeout{pgf_backend_query_start}
-""" % (latex_preamble, latex_fontspec)
+ latex_header = [r"\documentclass{minimal}",
+ latex_preamble,
+ latex_fontspec,
+ r"\begin{document}",
+ r"text $math \mu$", # force latex to load fonts now
+ r"\typeout{pgf_backend_query_start}"]
+ return "\n".join(latex_header)
+
+ def _stdin_writeln(self, s):
+ self.latex_stdin_utf8.write(s)
+ self.latex_stdin_utf8.write("\n")
+ self.latex_stdin_utf8.flush()
+
+ def _expect(self, s):
+ exp = s.encode("utf8")
+ buf = bytearray()
+ while True:
+ b = self.latex.stdout.read(1)
+ buf += b
+ if buf[-len(exp):] == exp:
+ break
+ if not len(b):
+ raise LatexError("LaTeX process halted", buf.decode("utf8"))
+ return buf.decode("utf8")
- return latex_header
+ def _expect_prompt(self):
+ return self._expect("\n*")
def __init__(self):
self.texcommand = get_texcommand()
@@ -238,27 +257,23 @@ def __init__(self):
# test the LaTeX setup to ensure a clean startup of the subprocess
latex = subprocess.Popen([self.texcommand, "-halt-on-error"],
stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- universal_newlines=True)
- stdout, stderr = latex.communicate(self.latex_header + latex_end)
+ stdout=subprocess.PIPE)
+ test_input = self.latex_header + latex_end
+ stdout, stderr = latex.communicate(test_input.encode("utf-8"))
if latex.returncode != 0:
raise LatexError("LaTeX returned an error, probably missing font or error in preamble:\n%s" % stdout)
- # open LaTeX process
+ # open LaTeX process for real work
latex = subprocess.Popen([self.texcommand, "-halt-on-error"],
stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- universal_newlines=True)
- latex.stdin.write(self.latex_header)
- latex.stdin.flush()
- # read all lines until our 'pgf_backend_query_start' token appears
- while not latex.stdout.readline().startswith("*pgf_backend_query_start"):
- pass
- while latex.stdout.read(1) != '*':
- pass
+ stdout=subprocess.PIPE)
self.latex = latex
- self.latex_stdin = codecs.getwriter("utf-8")(latex.stdin)
- self.latex_stdout = codecs.getreader("utf-8")(latex.stdout)
+ self.latex_stdin_utf8 = codecs.getwriter("utf8")(self.latex.stdin)
+ # write header with 'pgf_backend_query_start' token
+ self._stdin_writeln(self._build_latex_header())
+ # read all lines until our 'pgf_backend_query_start' token appears
+ self._expect("*pgf_backend_query_start")
+ self._expect_prompt()
# cache for strings already processed
self.str_cache = {}
@@ -267,8 +282,8 @@ def __del__(self):
if rcParams.get("pgf.debug", False):
print "deleting LatexManager"
try:
- self.latex.terminate()
- self.latex.wait()
+ self.latex_stdin_utf8.close()
+ self.latex.communicate()
except:
pass
try:
@@ -277,19 +292,6 @@ def __del__(self):
except:
pass
- def _wait_for_prompt(self):
- """
- Read all bytes from LaTeX stdout until a new line starts with a *.
- """
- buf = [""]
- while True:
- buf.append(self.latex_stdout.read(1))
- if buf[-1] == "*" and buf[-2] == "\n":
- break
- if buf[-1] == "":
- raise LatexError("LaTeX process halted", u"".join(buf))
- return "".join(buf)
-
def get_width_height_descent(self, text, prop):
"""
Get the width, total height and descent for a text typesetted by the
@@ -298,30 +300,27 @@ def get_width_height_descent(self, text, prop):
# apply font properties and define textbox
prop_cmds = _font_properties_str(prop)
- textbox = u"\\sbox0{%s %s}\n" % (prop_cmds, text)
+ textbox = "\\sbox0{%s %s}" % (prop_cmds, text)
# check cache
if textbox in self.str_cache:
return self.str_cache[textbox]
# send textbox to LaTeX and wait for prompt
- self.latex_stdin.write(unicode(textbox))
- self.latex_stdin.flush()
+ self._stdin_writeln(textbox)
try:
- self._wait_for_prompt()
+ self._expect_prompt()
except LatexError as e:
- msg = u"Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output)
+ msg = "Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output)
raise ValueError(msg)
# typeout width, height and text offset of the last textbox
- query = "\\typeout{\\the\\wd0,\\the\\ht0,\\the\\dp0}\n"
- self.latex_stdin.write(query)
- self.latex_stdin.flush()
+ self._stdin_writeln(r"\typeout{\the\wd0,\the\ht0,\the\dp0}")
# read answer from latex and advance to the next prompt
try:
- answer = self._wait_for_prompt()
+ answer = self._expect_prompt()
except LatexError as e:
- msg = u"Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output)
+ msg = "Error processing '%s'\nLaTeX Output:\n%s" % (text, e.latex_output)
raise ValueError(msg)
# parse metrics from the answer string
@@ -625,12 +624,7 @@ def __init__(self, *args):
def get_default_filetype(self):
return 'pdf'
- def print_pgf(self, filename, *args, **kwargs):
- """
- Output pgf commands for drawing the figure so it can be included and
- rendered in latex documents.
- """
-
+ def _print_pgf_to_fh(self, fh):
header_text = r"""%% Creator: Matplotlib, PGF backend
%%
%% To include the figure in your LaTeX document, write
@@ -660,37 +654,50 @@ def print_pgf(self, filename, *args, **kwargs):
# get figure size in inch
w, h = self.figure.get_figwidth(), self.figure.get_figheight()
- # start a pgfpicture environment and set a bounding box
- with codecs.open(filename, "w", encoding="utf-8") as fh:
- fh.write(header_text)
- fh.write(header_info_preamble)
- fh.write("\n")
- writeln(fh, r"\begingroup")
- writeln(fh, r"\makeatletter")
- writeln(fh, r"\begin{pgfpicture}")
- writeln(fh, r"\pgfpathrectangle{\pgfpointorigin}{\pgfqpoint{%fin}{%fin}}" % (w,h))
- writeln(fh, r"\pgfusepath{use as bounding box}")
-
- renderer = RendererPgf(self.figure, fh)
- self.figure.draw(renderer)
-
- # end the pgfpicture environment
- writeln(fh, r"\end{pgfpicture}")
- writeln(fh, r"\makeatother")
- writeln(fh, r"\endgroup")
-
- def print_pdf(self, filename, *args, **kwargs):
+ # create pgfpicture environment and write the pgf code
+ fh.write(header_text)
+ fh.write(header_info_preamble)
+ fh.write("\n")
+ writeln(fh, r"\begingroup")
+ writeln(fh, r"\makeatletter")
+ writeln(fh, r"\begin{pgfpicture}")
+ writeln(fh, r"\pgfpathrectangle{\pgfpointorigin}{\pgfqpoint{%fin}{%fin}}" % (w,h))
+ writeln(fh, r"\pgfusepath{use as bounding box}")
+ renderer = RendererPgf(self.figure, fh)
+ self.figure.draw(renderer)
+
+ # end the pgfpicture environment
+ writeln(fh, r"\end{pgfpicture}")
+ writeln(fh, r"\makeatother")
+ writeln(fh, r"\endgroup")
+
+ def print_pgf(self, fname_or_fh, *args, **kwargs):
"""
- Use LaTeX to compile a Pgf generated figure to PDF.
+ Output pgf commands for drawing the figure so it can be included and
+ rendered in latex documents.
"""
- w, h = self.figure.get_figwidth(), self.figure.get_figheight()
+ if kwargs.get("dryrun", False): return
+
+ # figure out where the pgf is to be written to
+ if is_string_like(fname_or_fh):
+ with codecs.open(fname_or_fh, "w", encoding="utf-8") as fh:
+ self._print_pgf_to_fh(fh)
+ elif is_writable_file_like(fname_or_fh):
+ raise ValueError("saving pgf to a stream is not supported, " + \
+ "consider using the pdf option of the pgf-backend")
+ else:
+ raise ValueError("filename must be a path")
- target = os.path.abspath(filename)
+ def _print_pdf_to_fh(self, fh):
+ w, h = self.figure.get_figwidth(), self.figure.get_figheight()
try:
+ # create and switch to temporary directory
tmpdir = tempfile.mkdtemp()
cwd = os.getcwd()
os.chdir(tmpdir)
+
+ # print figure to pgf and compile it with latex
self.print_pgf("figure.pgf")
latex_preamble = get_preamble()
@@ -706,16 +713,19 @@ def print_pdf(self, filename, *args, **kwargs):
\centering
\input{figure.pgf}
\end{document}""" % (w, h, latex_preamble, latex_fontspec)
- with codecs.open("figure.tex", "w", "utf-8") as fh:
- fh.write(latexcode)
+ with codecs.open("figure.tex", "w", "utf-8") as fh_tex:
+ fh_tex.write(latexcode)
texcommand = get_texcommand()
cmdargs = [texcommand, "-interaction=nonstopmode", "-halt-on-error", "figure.tex"]
try:
- stdout = subprocess.check_output(cmdargs, universal_newlines=True, stderr=subprocess.STDOUT)
- except:
- raise RuntimeError("%s was not able to process your file.\n\nFull log:\n%s" % (texcommand, stdout))
- shutil.copyfile("figure.pdf", target)
+ subprocess.check_output(cmdargs, stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ raise RuntimeError("%s was not able to process your file.\n\nFull log:\n%s" % (texcommand, e.output))
+
+ # copy file contents to target
+ with open("figure.pdf", "rb") as fh_src:
+ shutil.copyfileobj(fh_src, fh)
finally:
os.chdir(cwd)
try:
@@ -723,21 +733,33 @@ def print_pdf(self, filename, *args, **kwargs):
except:
sys.stderr.write("could not delete tmp directory %s\n" % tmpdir)
- def print_png(self, filename, *args, **kwargs):
+ def print_pdf(self, fname_or_fh, *args, **kwargs):
"""
- Use LaTeX to compile a pgf figure to pdf and convert it to png.
+ Use LaTeX to compile a Pgf generated figure to PDF.
"""
+ # figure out where the pdf is to be written to
+ if is_string_like(fname_or_fh):
+ with open(fname_or_fh, "wb") as fh:
+ self._print_pdf_to_fh(fh)
+ elif is_writable_file_like(fname_or_fh):
+ self._print_pdf_to_fh(fname_or_fh)
+ else:
+ raise ValueError("filename must be a path or a file-like object")
+ def _print_png_to_fh(self, fh):
converter = make_pdf_to_png_converter()
- target = os.path.abspath(filename)
try:
+ # create and switch to temporary directory
tmpdir = tempfile.mkdtemp()
cwd = os.getcwd()
os.chdir(tmpdir)
+ # create pdf and try to convert it to png
self.print_pdf("figure.pdf")
converter("figure.pdf", "figure.png", dpi=self.figure.dpi)
- shutil.copyfile("figure.png", target)
+ # copy file contents to target
+ with open("figure.png", "rb") as fh_src:
+ shutil.copyfileobj(fh_src, fh)
finally:
os.chdir(cwd)
try:
@@ -745,6 +767,18 @@ def print_png(self, filename, *args, **kwargs):
except:
sys.stderr.write("could not delete tmp directory %s\n" % tmpdir)
+ def print_png(self, fname_or_fh, *args, **kwargs):
+ """
+ Use LaTeX to compile a pgf figure to pdf and convert it to png.
+ """
+ if is_string_like(fname_or_fh):
+ with open(fname_or_fh, "wb") as fh:
+ self._print_png_to_fh(fh)
+ elif is_writable_file_like(fname_or_fh):
+ self._print_png_to_fh(fname_or_fh)
+ else:
+ raise ValueError("filename must be a path or a file-like object")
+
def _render_texts_pgf(self, fh):
# TODO: currently unused code path
View
41 lib/matplotlib/tests/test_backend_pgf.py
@@ -4,22 +4,32 @@
import shutil
import subprocess
import numpy as np
+import nose
+from nose.plugins.skip import SkipTest
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.testing.compare import compare_images, ImageComparisonFailure
-from matplotlib.testing.decorators import _image_directories, knownfailureif
+from matplotlib.testing.decorators import _image_directories
baseline_dir, result_dir = _image_directories(lambda: 'dummy func')
-def run(*args):
- try:
- subprocess.check_output(args)
- return True
- except:
- return False
+def check_for(texsystem):
+ header = r"""
+ \documentclass{minimal}
+ \usepackage{pgf}
+ \begin{document}
+ \typeout{pgfversion=\pgfversion}
+ \makeatletter
+ \@@end
+ """
+ latex = subprocess.Popen(["xelatex", "-halt-on-error"],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE)
+ stdout, stderr = latex.communicate(header.encode("utf8"))
+
+ return latex.returncode == 0
def switch_backend(backend):
- import nose
def switch_backend_decorator(func):
def backend_switcher(*args, **kwargs):
@@ -41,7 +51,7 @@ def compare_figure(fname):
expected = os.path.join(result_dir, "expected_%s" % fname)
shutil.copyfile(os.path.join(baseline_dir, fname), expected)
- err = compare_images(expected, actual, tol=1e-4)
+ err = compare_images(expected, actual, tol=5e-3)
if err:
raise ImageComparisonFailure('images not close: %s vs. %s' % (actual, expected))
@@ -58,9 +68,11 @@ def create_figure():
# test compiling a figure to pdf with xelatex
-@knownfailureif(not run('xelatex', '-v'), msg="xelatex is required for this test")
@switch_backend('pgf')
def test_xelatex():
+ if not check_for('xelatex'):
+ raise SkipTest('xelatex + pgf is required')
+
rc_xelatex = {'font.family': 'serif',
'pgf.rcfonts': False,}
mpl.rcParams.update(rc_xelatex)
@@ -69,9 +81,11 @@ def test_xelatex():
# test compiling a figure to pdf with pdflatex
-@knownfailureif(not run('pdflatex', '-v'), msg="pdflatex is required for this test")
@switch_backend('pgf')
def test_pdflatex():
+ if not check_for('pdflatex'):
+ raise SkipTest('pdflatex + pgf is required')
+
rc_pdflatex = {'font.family': 'serif',
'pgf.rcfonts': False,
'pgf.texsystem': 'pdflatex',
@@ -83,10 +97,11 @@ def test_pdflatex():
# test updating the rc parameters for each figure
-@knownfailureif(not run('pdflatex', '-v') or not run('xelatex', '-v'),
- msg="xelatex and pdflatex are required for this test")
@switch_backend('pgf')
def test_rcupdate():
+ if not check_for('xelatex') or not check_for('pdflatex'):
+ raise SkipTest('xelatex and pdflatex + pgf required')
+
rc_sets = []
rc_sets.append({'font.family': 'sans-serif',
'font.size': 30,
Something went wrong with that request. Please try again.