fix encoding bugs for publish

src/pyff/builtins.py

@@ -360,20 +360,17 @@ def publish(req, *opts):
         output_file = req.args[0]
     if output_file is not None:
         output_file = output_file.strip()
-        log.debug("publish {}".format(output_file))
         resource_name = output_file
         m = re.match(FILESPEC_REGEX, output_file)
         if m:
             output_file = m.group(1)
             resource_name = m.group(2)
-        log.debug("output_file={}, resource_name={}".format(output_file, resource_name))
         out = output_file
         if os.path.isdir(output_file):
             out = "{}.xml".format(os.path.join(output_file, req.id))
 
         data = dumptree(req.t)
-        if six.PY3:
-            data = data.decode('utf-8')
+
         safe_write(out, data)
         req.store.update(req.t, tid=resource_name)  # TODO maybe this is not the right thing to do anymore
     return req.t
@@ -1149,7 +1146,7 @@ def emit(req, ctype="application/xml", *opts):
 
     req.state['headers']['Content-Type'] = ctype
     if six.PY2:
-        d = six.u(d.decode('utf-8')).encode("utf-8")
+        d = six.u(d)
     return d
 
 

src/pyff/utils.py

@@ -36,6 +36,7 @@
 import time
 from markupsafe import Markup
 import six
+import traceback
 from . import __version__
 
 etree.set_default_parser(etree.XMLParser(resolve_entities=False))
@@ -241,18 +242,30 @@ def safe_write(fn, data):
     """
     tmpn = None
     try:
-        log.debug("writing {} chrs into {}".format(len(data), fn))
         fn = os.path.expanduser(fn)
         dirname, basename = os.path.split(fn)
-        with tempfile.NamedTemporaryFile('w', delete=False, prefix=".%s" % basename, dir=dirname) as tmp:
-            if isinstance(data, six.binary_type):
-                data = data.decode('utf-8')
+        kwargs = dict(delete=False, prefix=".%s" % basename, dir=dirname)
+        if six.PY3:
+            kwargs['encoding'] = "utf-8"
+            mode = 'w+'
+        else:
+            mode = 'w+b'
+
+        if isinstance(data, six.binary_type):
+            data = data.decode('utf-8')
+
+        with tempfile.NamedTemporaryFile(mode, **kwargs) as tmp:
+            if six.PY2:
+                data = data.encode('utf-8')
+
+            log.debug("safe writing {} chrs into {}".format(len(data), fn))
             tmp.write(data)
             tmpn = tmp.name
         if os.path.exists(tmpn) and os.stat(tmpn).st_size > 0:
             os.rename(tmpn, fn)
             return True
     except Exception as ex:
+        log.debug(traceback.format_exc())
         log.error(ex)
     finally:
         if tmpn is not None and os.path.exists(tmpn):
@@ -267,6 +280,7 @@ def safe_write(fn, data):
 env = Environment(loader=PackageLoader(__package__, 'templates'), extensions=['jinja2.ext.i18n'])
 getattr(env, 'install_gettext_callables')(language.gettext, language.ngettext, newstyle=True)
 
+
 def urlencode_filter(s):
     if type(s) == 'Markup':
         s = s.unescape()
@@ -606,8 +620,6 @@ def url_get(url):
     s = None
     info = dict()
 
-    log.debug("GET URL {!s}".format(url))
-
     if 'file://' in url:
         s = requests.session()
         s.mount('file://', FileAdapter())
@@ -624,6 +636,11 @@ def url_get(url):
         s = requests.Session()
         r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout)
 
+    if six.PY2:
+        r.encoding = "utf-8"
+
+    log.debug("url_get({}) returns {} chrs encoded as {}".format(url, len(r.content), r.encoding))
+
     if config.request_override_encoding is not None:
         r.encoding = config.request_override_encoding