Skip to content
Permalink
Browse files

core: eliminate some quadratric behaviour from IoLogger

This is the same problem as used to afflict Stream: large input buffers
containing many small messages cause intense string copying. This
eliminates the worst of it.
  • Loading branch information...
dw committed Mar 9, 2019
1 parent 77564fd commit a5536c3514517af3bc10260e880188cdedcbcacd
Showing with 13 additions and 7 deletions.
  1. +13 −7 mitogen/core.py
@@ -2405,7 +2405,7 @@ class IoLogger(BasicStream):
:class:`BasicStream` subclass that sets up redirection of a standard
UNIX file descriptor back into the Python :mod:`logging` package.
"""
_buf = ''
_trailer = u''

def __init__(self, broker, name, dest_fd):
self._broker = broker
@@ -2427,10 +2427,17 @@ def __init__(self, broker, name, dest_fd):
def __repr__(self):
return '<IoLogger %s>' % (self._name,)

def _log_lines(self):
while self._buf.find('\n') != -1:
line, _, self._buf = str_partition(self._buf, '\n')
self._log.info('%s', line.rstrip('\n'))
def _log_lines(self, buf):
start = 0
while True:
nl = min(buf.find('\n', start), start+1024)
if nl == -1:
break
self._log.info('%s', buf[start:nl])
start = nl + 1

if start:
self._trailer = buf[start:]

def on_shutdown(self, broker):
"""Shut down the write end of the logging socket."""
@@ -2447,8 +2454,7 @@ def on_receive(self, broker):
if not buf:
return self.on_disconnect(broker)

self._buf += buf.decode('latin1')
self._log_lines()
self._log_lines(self._trailer + buf.decode('latin1'))


class Router(object):

0 comments on commit a5536c3

Please sign in to comment.
You can’t perform that action at this time.