Make text_factory for PY2 (lektor#299)

Make sqlite text_factory for Python 2

So that we avoid problems with non-ASCII characters in file names.

lektor/builder.py

@@ -1,3 +1,4 @@
+import locale
 import os
 import sys
 import stat
@@ -12,7 +13,7 @@
 
 from werkzeug.posixemulation import rename
 
-from lektor._compat import iteritems, text_type
+from lektor._compat import PY2, iteritems, text_type
 from lektor.context import Context
 from lektor.build_programs import builtin_build_programs
 from lektor.reporter import reporter
@@ -984,6 +985,20 @@ def buildstate_database_filename(self):
     def connect_to_database(self):
         con = sqlite3.connect(self.buildstate_database_filename,
                               timeout=10, check_same_thread=False)
+        if PY2:
+            # This code block solve lektor/lektor#243 issue
+            # `os.walk` return :class:`str` type string. But :class:`str` mean
+            # diffrent type between Python 2 and 3.
+            # (:class:`str` on PY2 is equivalent to :class:`bytes` on PY3)
+            # :mod:`sqlite` can not consume multibyte input string because
+            # it expect :class:`unicode` on PY2, not bytes.
+            # If sqlite make connection without text_factory, Multibyte
+            # filename must raise :class:`ProgrammingError`
+            # So we must decode filename as system encoding for on PY2 via
+            # text_factory.
+
+            system_encoding = locale.getdefaultlocale()[1]
+            con.text_factory = lambda x: x.decode(system_encoding, 'ignore')
         cur = con.cursor()
         cur.execute('pragma journal_mode=WAL')
         cur.execute('pragma synchronous=NORMAL')

tests/test_unicode.py

@@ -23,6 +23,21 @@ def test_unicode_project_folder(tmpdir):
         assert f.read() == b'<h1>Hello</h1>\n<p>W\xc3\xb6rld</p>\n\n'
 
 
+def test_unicode_attachment_filename(tmpdir):
+    from lektor.reporter import BufferReporter
+
+    pad, builder = get_unicode_builder(tmpdir)
+
+    with BufferReporter(builder.env) as reporter:
+        prog, _ = builder.build(pad.root.attachments.first())
+
+        failures = reporter.get_failures()
+        assert len(failures) == 0
+
+        with prog.artifacts[0].open('rb') as f:
+            assert f.read() == b'attachment\n'
+
+
 def test_bad_file_ignored(tmpdir):
     from lektor.reporter import BufferReporter
     from lektor.build_programs import BuildError

tests/ünicöde-project/content/ättachment

@@ -0,0 +1 @@
+attachment