Merge pull request Supervisor#1235 from vsajip/fix-1231

Initial cut for fix of issue Supervisor#1231

Author: mnaberez

.gitignore

@@ -1,5 +1,4 @@
 *~
-*.conf
 *.egg
 *.egg-info
 *.log

supervisor/http_client.py

@@ -29,10 +29,23 @@ def done(self, url):
 
     def feed(self, url, data):
         try:
-            data = as_string(data)
+            sdata = as_string(data)
         except UnicodeDecodeError:
-            data = 'Undecodable: %r' % data
-        sys.stdout.write(data)
+            sdata = 'Undecodable: %r' % data
+        # We've got Unicode data in sdata now, but writing to stdout sometimes
+        # fails - see issue #1231.
+        try:
+            sys.stdout.write(sdata)
+        except UnicodeEncodeError as e:
+            if sys.version_info[0] < 3:
+                # This might seem like The Wrong Thing To Do (writing bytes
+                # rather than text to an output stream), but it seems to work
+                # OK for Python 2.7.
+                sys.stdout.write(data)
+            else:
+                s = ('Unable to write Unicode to stdout because it has '
+                     'encoding %s' % sys.stdout.encoding)
+                raise ValueError(s)
         sys.stdout.flush()
 
     def close(self, url):

supervisor/supervisorctl.py

@@ -440,6 +440,14 @@ class DefaultControllerPlugin(ControllerPluginBase):
     name = 'default'
     listener = None # for unit tests
     def _tailf(self, path):
+        def not_all_langs():
+            enc = getattr(sys.stdout, 'encoding', '').lower()
+            return None if enc.startswith('utf') else sys.stdout.encoding
+
+        problematic_enc = not_all_langs()
+        if problematic_enc:
+            self.ctl.output('Warning: sys.stdout.encoding is set to %s, so '
+                            'Unicode output may fail.' % problematic_enc)
         self.ctl.output('==> Press Ctrl-C to exit <==')
 
         username = self.ctl.options.username

supervisor/tests/fixtures/issue-1231.conf

@@ -0,0 +1,17 @@
+[supervisord]
+loglevel=info                ; log level; default info; others: debug,warn,trace
+logfile=/tmp/issue-1231.log  ; main log file; default $CWD/supervisord.log
+pidfile=/tmp/issue-1231.pid  ; supervisord pidfile; default supervisord.pid
+nodaemon=true                ; start in foreground if true; default false
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[unix_http_server]
+file=/tmp/issue-1231.sock    ; the path to the socket file
+
+[supervisorctl]
+serverurl=unix:///tmp/issue-1231.sock   ; use a unix:// URL  for a unix socket
+
+[program:hello]
+command=python %(here)s/test_1231.py

supervisor/tests/fixtures/test_1231.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+import logging
+import random
+import sys
+import time
+
+def main():
+    logging.basicConfig(level=logging.INFO, stream=sys.stdout,
+                        format='%(levelname)s [%(asctime)s] %(message)s',
+                        datefmt='%m-%d|%H:%M:%S')
+    i = 1
+    while True:
+        delay = random.randint(400, 1200)
+        time.sleep(delay / 1000.0)
+        logging.info('%d - hash=57d94b…381088', i)
+        i += 1
+
+if __name__ == '__main__':
+    main()

supervisor/tests/test_end_to_end.py

@@ -136,6 +136,54 @@ def test_issue_1224(self):
         self.addCleanup(supervisord.kill, signal.SIGINT)
         supervisord.expect_exact('cat entered RUNNING state', timeout=60)
 
+    def test_issue_1231a(self):
+        filename = pkg_resources.resource_filename(__name__, 'fixtures/issue-1231.conf')
+        args = ['-m', 'supervisor.supervisord', '-c', filename]
+        supervisord = pexpect.spawn(sys.executable, args, encoding='utf-8')
+        self.addCleanup(supervisord.kill, signal.SIGINT)
+        supervisord.expect_exact('success: hello entered RUNNING state')
+
+        args = ['-m', 'supervisor.supervisorctl', '-c', filename, 'tail', '-f', 'hello']
+        supervisorctl = pexpect.spawn(sys.executable, args, encoding='utf-8')
+        self.addCleanup(supervisorctl.kill, signal.SIGINT)
+
+        for i in range(1, 4):
+            line = '%d - hash=57d94b…381088' % i
+            supervisorctl.expect_exact(line, timeout=30)
+
+
+    def test_issue_1231b(self):
+        filename = pkg_resources.resource_filename(__name__, 'fixtures/issue-1231.conf')
+        args = ['-m', 'supervisor.supervisord', '-c', filename]
+        supervisord = pexpect.spawn(sys.executable, args, encoding='utf-8')
+        self.addCleanup(supervisord.kill, signal.SIGINT)
+        supervisord.expect_exact('success: hello entered RUNNING state')
+
+        args = ['-m', 'supervisor.supervisorctl', '-c', filename, 'tail', '-f', 'hello']
+        env = os.environ.copy()
+        env['LANG'] = 'oops'
+        supervisorctl = pexpect.spawn(sys.executable, args, encoding='utf-8',
+                                      env=env)
+        self.addCleanup(supervisorctl.kill, signal.SIGINT)
+
+        # For Python 3 < 3.7, LANG=oops leads to warnings because of the
+        # stdout encoding. For 3.7 (and presumably later), the encoding is
+        # utf-8 when LANG=oops.
+        if sys.version_info[:2] < (3, 7):
+            supervisorctl.expect('Warning: sys.stdout.encoding is set to ',
+                                 timeout=30)
+            supervisorctl.expect('Unicode output may fail.', timeout=30)
+
+        for i in range(1, 4):
+            line = '%d - hash=57d94b…381088' % i
+            try:
+                supervisorctl.expect_exact(line, timeout=30)
+            except pexpect.exceptions.EOF:
+                self.assertIn('Unable to write Unicode to stdout because it '
+                              'has encoding ',
+                              supervisorctl.before)
+                break
+
 
 def test_suite():
     return unittest.findTestCases(sys.modules[__name__])