Merge pull request #2433 from mdboom/unicode-font-filenames

Handle Unicode font filenames correctly/Fix crashing MacOSX backend

lib/matplotlib/backends/backend_agg.py

@@ -262,7 +262,7 @@ def _get_agg_font(self, prop):
             font = RendererAgg._fontd.get(fname)
             if font is None:
                 font = FT2Font(
-                    str(fname),
+                    fname,
                     hinting_factor=rcParams['text.hinting_factor'])
                 RendererAgg._fontd[fname] = font
             RendererAgg._fontd[key] = font

lib/matplotlib/backends/backend_pdf.py

@@ -585,7 +585,7 @@ def fontName(self, fontprop):
             self.fontNames[filename] = Fx
             self.nextFont += 1
             matplotlib.verbose.report(
-                'Assigning font %s = %s' % (Fx, filename),
+                'Assigning font %s = %r' % (Fx, filename),
                 'debug')
 
         return Fx
@@ -701,7 +701,7 @@ def createType1Descriptor(self, t1font, fontfile):
         if 0:             flags |= 1 << 17 # TODO: small caps
         if 0:             flags |= 1 << 18 # TODO: force bold
 
-        ft2font = FT2Font(str(fontfile))
+        ft2font = FT2Font(fontfile)
 
         descriptor = {
             'Type':        Name('FontDescriptor'),
@@ -761,7 +761,7 @@ def _get_xobject_symbol_name(self, filename, symbol_name):
     def embedTTF(self, filename, characters):
         """Embed the TTF font from the named file into the document."""
 
-        font = FT2Font(str(filename))
+        font = FT2Font(filename)
         fonttype = rcParams['pdf.fonttype']
 
         def cvt(length, upe=font.units_per_EM, nearest=True):
@@ -845,7 +845,8 @@ def get_char_width(charcode):
 
             # Make the charprocs array (using ttconv to generate the
             # actual outlines)
-            rawcharprocs = ttconv.get_pdf_charprocs(filename, glyph_ids)
+            rawcharprocs = ttconv.get_pdf_charprocs(
+                filename.encode(sys.getfilesystemencoding()), glyph_ids)
             charprocs = {}
             charprocsRef = {}
             for charname, stream in six.iteritems(rawcharprocs):
@@ -2003,7 +2004,7 @@ def _get_font_ttf(self, prop):
             filename = findfont(prop)
             font = self.truetype_font_cache.get(filename)
             if font is None:
-                font = FT2Font(str(filename))
+                font = FT2Font(filename)
                 self.truetype_font_cache[filename] = font
             self.truetype_font_cache[key] = font
         font.clear()

lib/matplotlib/backends/backend_pgf.py

@@ -33,7 +33,7 @@
 system_fonts = []
 for f in font_manager.findSystemFonts():
     try:
-        system_fonts.append(FT2Font(str(f)).family_name)
+        system_fonts.append(FT2Font(f).family_name)
     except RuntimeError:
         pass  # some fonts on osx are known to fail, print?
     except:

lib/matplotlib/backends/backend_ps.py

@@ -391,7 +391,7 @@ def _get_font_ttf(self, prop):
             fname = findfont(prop)
             font = self.fontd.get(fname)
             if font is None:
-                font = FT2Font(str(fname))
+                font = FT2Font(fname)
                 self.fontd[fname] = font
             self.fontd[key] = font
         font.clear()
@@ -1131,7 +1131,7 @@ def print_figure_impl():
             if not rcParams['ps.useafm']:
                 for font_filename, chars in six.itervalues(ps_renderer.used_characters):
                     if len(chars):
-                        font = FT2Font(str(font_filename))
+                        font = FT2Font(font_filename)
                         cmap = font.get_charmap()
                         glyph_ids = []
                         for c in chars:
@@ -1153,7 +1153,9 @@ def print_figure_impl():
                             raise RuntimeError("OpenType CFF fonts can not be saved using the internal Postscript backend at this time.\nConsider using the Cairo backend.")
                         else:
                             fh.flush()
-                            convert_ttf_to_ps(font_filename, raw_fh, fonttype, glyph_ids)
+                            convert_ttf_to_ps(
+                                font_filename.encode(sys.getfilesystemencoding()),
+                                raw_fh, fonttype, glyph_ids)
             print("end", file=fh)
             print("%%EndProlog", file=fh)
 

lib/matplotlib/backends/backend_svg.py

@@ -323,7 +323,7 @@ def _get_font(self, prop):
             fname = findfont(prop)
             font = self.fontd.get(fname)
             if font is None:
-                font = FT2Font(str(fname))
+                font = FT2Font(fname)
                 self.fontd[fname] = font
             self.fontd[key] = font
         font.clear()

lib/matplotlib/font_manager.py

@@ -62,7 +62,6 @@
     parse_fontconfig_pattern, generate_fontconfig_pattern
 
 USE_FONTCONFIG = False
-
 verbose = matplotlib.verbose
 
 font_scalings = {
@@ -269,16 +268,20 @@ def get_fontconfig_fonts(fontext='ttf'):
 
     fontfiles = {}
     try:
-        pipe = subprocess.Popen(['fc-list', '', 'file'], stdout=subprocess.PIPE)
+        pipe = subprocess.Popen(['fc-list', '--format=%{file}\\n'], stdout=subprocess.PIPE)
         output = pipe.communicate()[0]
     except (OSError, IOError):
         # Calling fc-list did not work, so we'll just return nothing
         return fontfiles
 
     if pipe.returncode == 0:
-        output = str(output)
-        for line in output.split('\n'):
-            fname = line.split(':')[0]
+        # The line breaks between results are in ascii, but each entry
+        # is in in sys.filesystemencoding().
+        for fname in output.split(b'\n'):
+            try:
+                fname = six.text_type(fname, sys.getfilesystemencoding())
+            except UnicodeDecodeError:
+                continue
             if (os.path.splitext(fname)[1][1:] in fontext and
                 os.path.exists(fname)):
                 fontfiles[fname] = 1
@@ -570,7 +573,7 @@ def createFontList(fontfiles, fontext='ttf'):
                 continue
         else:
             try:
-                font = ft2font.FT2Font(str(fpath))
+                font = ft2font.FT2Font(fpath)
             except RuntimeError:
                 verbose.report("Could not open font file %s"%fpath)
                 continue
@@ -720,7 +723,7 @@ def get_name(self):
         Return the name of the font that best matches the font
         properties.
         """
-        return ft2font.FT2Font(str(findfont(self))).family_name
+        return ft2font.FT2Font(findfont(self)).family_name
 
     def get_style(self):
         """
@@ -1246,7 +1249,7 @@ def findfont(self, prop, fontext='ttf', directory=None,
         else:
             verbose.report(
                 'findfont: Matching %s to %s (%s) with score of %f' %
-                (prop, best_font.name, best_font.fname, best_score))
+                (prop, best_font.name, repr(best_font.fname), best_score))
             result = best_font.fname
 
         if not os.path.isfile(result):
@@ -1292,19 +1295,26 @@ def fc_match(pattern, fontext):
         fontexts = get_fontext_synonyms(fontext)
         ext = "." + fontext
         try:
-            pipe = subprocess.Popen(['fc-match', '-sv', pattern], stdout=subprocess.PIPE)
+            pipe = subprocess.Popen(
+                ['fc-match', '-s', '--format=%{file}\\n', pattern],
+                stdout=subprocess.PIPE)
             output = pipe.communicate()[0]
         except (OSError, IOError):
             return None
+
+        # The bulk of the output from fc-list is ascii, so we keep the
+        # result in bytes and parse it as bytes, until we extract the
+        # filename, which is in sys.filesystemencoding().
         if pipe.returncode == 0:
-            for match in _fc_match_regex.finditer(output):
-                file = match.group(1)
-                file = file.decode(sys.getfilesystemencoding())
-                if os.path.splitext(file)[1][1:] in fontexts:
-                    return file
+            for fname in output.split(b'\n'):
+                try:
+                    fname = six.text_type(fname, sys.getfilesystemencoding())
+                except UnicodeDecodeError:
+                    continue
+                if os.path.splitext(fname)[1][1:] in fontexts:
+                    return fname
         return None
 
-    _fc_match_regex = re.compile(br'\sfile:\s+"([^"]*)"')
     _fc_match_cache = {}
 
     def findfont(prop, fontext='ttf'):

lib/matplotlib/mathtext.py

@@ -566,7 +566,7 @@ def __init__(self, default_font_prop, mathtext_backend):
         self._fonts = {}
 
         filename = findfont(default_font_prop)
-        default_font = self.CachedFont(FT2Font(str(filename)))
+        default_font = self.CachedFont(FT2Font(filename))
         self._fonts['default'] = default_font
         self._fonts['regular'] = default_font
 
@@ -581,8 +581,8 @@ def _get_font(self, font):
             basename = font
 
         cached_font = self._fonts.get(basename)
-        if cached_font is None:
-            font = FT2Font(str(basename))
+        if cached_font is None and os.path.exists(basename):
+            font = FT2Font(basename)
             cached_font = self.CachedFont(font)
             self._fonts[basename] = cached_font
             self._fonts[font.postscript_name] = cached_font
@@ -697,20 +697,14 @@ def _get_glyph(self, fontname, font_class, sym, fontsize):
         if fontname in self.fontmap and sym in latex_to_bakoma:
             basename, num = latex_to_bakoma[sym]
             slanted = (basename == "cmmi10") or sym in self._slanted_symbols
-            try:
-                cached_font = self._get_font(basename)
-            except RuntimeError:
-                pass
-            else:
+            cached_font = self._get_font(basename)
+            if cached_font is not None:
                 symbol_name = cached_font.font.get_glyph_name(num)
                 num = cached_font.glyphmap[num]
         elif len(sym) == 1:
             slanted = (fontname == "it")
-            try:
-                cached_font = self._get_font(fontname)
-            except RuntimeError:
-                pass
-            else:
+            cached_font = self._get_font(fontname)
+            if cached_font is not None:
                 num = ord(sym)
                 gid = cached_font.charmap.get(num)
                 if gid is not None:
@@ -852,11 +846,8 @@ def _get_glyph(self, fontname, font_class, sym, fontsize):
 
             slanted = (new_fontname == 'it') or sym in self._slanted_symbols
             found_symbol = False
-            try:
-                cached_font = self._get_font(new_fontname)
-            except RuntimeError:
-                pass
-            else:
+            cached_font = self._get_font(new_fontname)
+            if cached_font is not None:
                 try:
                     glyphindex = cached_font.charmap[uniindex]
                     found_symbol = True

lib/matplotlib/textpath.py

@@ -56,7 +56,7 @@ def _get_font(self, prop):
         find a ttf font.
         """
         fname = font_manager.findfont(prop)
-        font = FT2Font(str(fname))
+        font = FT2Font(fname)
         font.set_size(self.FONT_SCALE, self.DPI)
 
         return font
@@ -338,7 +338,7 @@ def get_glyphs_tex(self, prop, s, glyph_map=None,
             font_bunch = self.tex_font_map[dvifont.texname]
 
             if font_and_encoding is None:
-                font = FT2Font(str(font_bunch.filename))
+                font = FT2Font(font_bunch.filename)
 
                 for charmap_name, charmap_code in [("ADOBE_CUSTOM",
                                                     1094992451),