support multibyte filename handling.

https://bitbucket.org/birkenfeld/sphinx/issue/703

Author: shimizukawa

sphinx/builders/epub.py

@@ -662,7 +662,12 @@ def build_epub(self, outdir, outname):
             zipfile.ZIP_STORED)
         for file in projectfiles:
             fp = path.join(outdir, file)
-            if isinstance(fp, unicode):
-                fp = fp.encode(sys.getfilesystemencoding())
+            if sys.version_info < (2, 6):
+                # When zipile.ZipFile.write call with unicode filename, ZipFile
+                # encode filename to 'utf-8' (only after Python-2.6).
+                if isinstance(file, unicode):
+                    # OEBPS Container Format (OCF) 2.0.1 specification require
+                    # "File Names MUST be UTF-8 encoded".
+                    file = file.encode('utf-8')
             epub.write(fp, file, zipfile.ZIP_DEFLATED)
         epub.close()

sphinx/cmdline.py

@@ -22,9 +22,17 @@
 from sphinx.application import Sphinx
 from sphinx.util import Tee, format_exception_cut_frames, save_traceback
 from sphinx.util.console import red, nocolor, color_terminal
+from sphinx.util.osutil import fs_encoding
 from sphinx.util.pycompat import terminal_safe, bytes
 
 
+def abspath(pathdir):
+    pathdir = path.abspath(pathdir)
+    if isinstance(pathdir, bytes):
+        pathdir = pathdir.decode(fs_encoding)
+    return pathdir
+
+
 def usage(argv, msg=None):
     if msg:
         print >>sys.stderr, msg
@@ -65,7 +73,7 @@ def main(argv):
     try:
         opts, args = getopt.getopt(argv[1:], 'ab:t:d:c:CD:A:ng:NEqQWw:P')
         allopts = set(opt[0] for opt in opts)
-        srcdir = confdir = path.abspath(args[0])
+        srcdir = confdir = abspath(args[0])
         if not path.isdir(srcdir):
             print >>sys.stderr, 'Error: Cannot find source directory `%s\'.' % (
                                 srcdir,)
@@ -75,7 +83,7 @@ def main(argv):
             print >>sys.stderr, ('Error: Source directory doesn\'t '
                                  'contain conf.py file.')
             return 1
-        outdir = path.abspath(args[1])
+        outdir = abspath(args[1])
         if not path.isdir(outdir):
             print >>sys.stderr, 'Making output directory...'
             os.makedirs(outdir)
@@ -119,9 +127,9 @@ def main(argv):
         elif opt == '-t':
             tags.append(val)
         elif opt == '-d':
-            doctreedir = path.abspath(val)
+            doctreedir = abspath(val)
         elif opt == '-c':
-            confdir = path.abspath(val)
+            confdir = abspath(val)
             if not path.isfile(path.join(confdir, 'conf.py')):
                 print >>sys.stderr, ('Error: Configuration directory '
                                      'doesn\'t contain conf.py file.')

sphinx/config.py

@@ -16,7 +16,7 @@
 
 from sphinx.errors import ConfigError
 from sphinx.locale import l_
-from sphinx.util.osutil import make_filename
+from sphinx.util.osutil import make_filename, fs_encoding
 from sphinx.util.pycompat import bytes, b, convert_with_2to3
 
 nonascii_re = re.compile(b(r'[\x80-\xff]'))
@@ -208,14 +208,15 @@ def __init__(self, dirname, filename, overrides, tags):
                     f.close()
                 try:
                     # compile to a code object, handle syntax errors
+                    config_file_enc = config_file.encode(fs_encoding)
                     try:
-                        code = compile(source, config_file, 'exec')
+                        code = compile(source, config_file_enc, 'exec')
                     except SyntaxError:
                         if convert_with_2to3:
                             # maybe the file uses 2.x syntax; try to refactor to
                             # 3.x syntax using 2to3
                             source = convert_with_2to3(config_file)
-                            code = compile(source, config_file, 'exec')
+                            code = compile(source, config_file_enc, 'exec')
                         else:
                             raise
                     exec code in config

sphinx/environment.py

@@ -41,15 +41,15 @@
      FilenameUniqDict
 from sphinx.util.nodes import clean_astext, make_refnode, extract_messages, \
      WarningStream
-from sphinx.util.osutil import movefile, SEP, ustrftime, find_catalog
+from sphinx.util.osutil import movefile, SEP, ustrftime, find_catalog, \
+     fs_encoding
 from sphinx.util.matching import compile_matchers
 from sphinx.util.pycompat import all, class_types
 from sphinx.util.websupport import is_commentable
 from sphinx.errors import SphinxError, ExtensionError
 from sphinx.locale import _, init as init_locale
 from sphinx.versioning import add_uids, merge_doctrees
 
-fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
 
 orig_role_function = roles.role
 orig_directive_function = directives.directive
@@ -1321,7 +1321,7 @@ def _walk_depth(node, depth, maxdepth):
         def _entries_from_toctree(toctreenode, parents,
                                   separate=False, subtree=False):
             """Return TOC entries for a toctree node."""
-            refs = [(e[0], str(e[1])) for e in toctreenode['entries']]
+            refs = [(e[0], e[1]) for e in toctreenode['entries']]
             entries = []
             for (title, ref) in refs:
                 try:

sphinx/util/osutil.py

@@ -148,3 +148,6 @@ def find_catalog(docname, compaction):
         ret = docname
 
     return ret
+
+fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
+

tests/path.py

@@ -16,16 +16,16 @@
 FILESYSTEMENCODING = sys.getfilesystemencoding() or sys.getdefaultencoding()
 
 
-class path(str):
+class path(unicode):
     """
     Represents a path which behaves like a string.
     """
     if sys.version_info < (3, 0):
         def __new__(cls, s, encoding=FILESYSTEMENCODING, errors='strict'):
-            if isinstance(s, unicode):
-                s = s.encode(encoding, errors=errors)
-                return str.__new__(cls, s)
-            return str.__new__(cls, s)
+            if isinstance(s, str):
+                s = s.decode(encoding, errors)
+                return unicode.__new__(cls, s)
+            return unicode.__new__(cls, s)
 
     @property
     def parent(self):
@@ -193,4 +193,4 @@ def joinpath(self, *args):
     __div__ = __truediv__ = joinpath
 
     def __repr__(self):
-        return '%s(%s)' % (self.__class__.__name__, str.__repr__(self))
+        return '%s(%s)' % (self.__class__.__name__, unicode.__repr__(self))

tests/test_build.py

@@ -10,6 +10,7 @@
 """
 
 from util import *
+from textwrap import dedent
 
 
 def teardown_module():
@@ -61,3 +62,22 @@ def test_man(app):
 @with_app(buildername='singlehtml', cleanenv=True)
 def test_singlehtml(app):
     app.builder.build_all()
+
+@with_app(buildername='html', srcdir='(temp)')
+def test_multibyte_path(app):
+    srcdir = path(app.srcdir)
+    mb_name = u'\u65e5\u672c\u8a9e'
+    (srcdir / mb_name).makedirs()
+    (srcdir / mb_name / (mb_name + '.txt')).write_text(dedent("""
+        multi byte file name page
+        ==========================
+        """))
+
+    master_doc = srcdir / 'contents.txt'
+    master_doc.write_bytes((master_doc.text() + dedent("""
+            .. toctree::
+            
+               %(mb_name)s/%(mb_name)s
+            """ % locals())
+    ).encode('utf-8'))
+    app.builder.build_all()