Skip to content

Commit

Permalink
master: cache sent/forwarded module names
Browse files Browse the repository at this point in the history
On 32x Docker run of issue_140__thread_pileup.yml

Before:

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
   384807    3.595    0.000   12.207    0.000 /home/dmw/src/mitogen/mitogen/master.py:896(_forward_one_module)
  1218352    4.867    0.000    7.302    0.000 /home/dmw/src/mitogen/mitogen/master.py:853(_send_module_and_related)

After:

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
   384807    3.839    0.000    6.543    0.000 /home/dmw/src/mitogen/mitogen/master.py:902(_forward_one_module)
  1218352    0.723    0.000    0.898    0.000 /home/dmw/src/mitogen/mitogen/master.py:856(_send_module_and_related)
  • Loading branch information
dw committed Jan 29, 2019
1 parent 49a8745 commit 0979422
Showing 1 changed file with 14 additions and 0 deletions.
14 changes: 14 additions & 0 deletions mitogen/master.py
Original file line number Diff line number Diff line change
Expand Up @@ -707,6 +707,9 @@ def __init__(self, router):
self.blacklist = []
self.whitelist = ['']

#: Context -> set([fullname, ..])
self._forwarded_by_context = {}

#: Number of GET_MODULE messages received.
self.get_module_count = 0
#: Total time spent in uncached GET_MODULE.
Expand Down Expand Up @@ -846,6 +849,9 @@ def _send_module_load_failed(self, stream, fullname):
)

def _send_module_and_related(self, stream, fullname):
if fullname in stream.sent_modules:
return

try:
tup = self._build_tuple(fullname)
for name in tup[4]: # related
Expand Down Expand Up @@ -889,6 +895,14 @@ def _send_forward_module(self, stream, context, fullname):
)

def _forward_one_module(self, context, fullname):
forwarded = self._forwarded_by_context.get(context)
if forwarded is None:
forwarded = set()
self._forwarded_by_context[context] = forwarded

if fullname in forwarded:
return

path = []
while fullname:
path.append(fullname)
Expand Down

0 comments on commit 0979422

Please sign in to comment.