exception on worker termination

[jreback]

getting an occasional exception on remote worker termination. I am using a slightly non-standard method as I have an external way of starting / stopping the actual worker processes.

This is a funtion on the Client

    def stop_workers(self):
        # close the remote workers
        try:
            sync(self.loop, self.scheduler.broadcast, msg={'op': 'terminate'})
        except tornado.iostream.StreamClosedError:
            pass

        # remove any workers that are left
        workers = self.ncores().keys()
        for w in workers:
            try:
                sync(self.loop, self.scheduler.unregister, address=w)
            except:
                pass

this sends a close to the workers, and removes any from the scheduler.

exception is this:

distributed.utils - ERROR - Stream is closed
Traceback (most recent call last):
  File "C:\Anaconda\envs\bmc3\lib\site-packages\distributed\utils.py", line 120, in f
    result[0] = yield gen.maybe_future(func(*args, **kwargs))
  File "C:\Anaconda\envs\bmc3\lib\site-packages\tornado\gen.py", line 1015, in run
    value = future.result()
  File "C:\Anaconda\envs\bmc3\lib\site-packages\tornado\concurrent.py", line 237, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 3, in raise_exc_info
  File "C:\Anaconda\envs\bmc3\lib\site-packages\tornado\gen.py", line 1021, in run
    yielded = self.gen.throw(*exc_info)
  File "C:\Anaconda\envs\bmc3\lib\site-packages\distributed\core.py", line 427, in send_recv_from_rpc
    deserialize=self.deserialize, **kwargs)
  File "C:\Anaconda\envs\bmc3\lib\site-packages\tornado\gen.py", line 1015, in run
    value = future.result()
  File "C:\Anaconda\envs\bmc3\lib\site-packages\tornado\concurrent.py", line 237, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 3, in raise_exc_info
  File "C:\Anaconda\envs\bmc3\lib\site-packages\tornado\gen.py", line 1024, in run
    yielded = self.gen.send(value)
  File "C:\Anaconda\envs\bmc3\lib\site-packages\distributed\core.py", line 310, in send_recv
    six.reraise(*clean_exception(**response))
  File "C:\Anaconda\envs\bmc3\lib\site-packages\six.py", line 658, in reraise
    raise value.with_traceback(tb)
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/distributed/core.py", line 172, in handle_stream
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
  File "<string>", line 3, in raise_exc_info
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1021, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/distributed/scheduler.py", line 1333, in broadcast
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
  File "<string>", line 3, in raise_exc_info
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1021, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/distributed/utils.py", line 98, in All
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
  File "<string>", line 3, in raise_exc_info
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1021, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/distributed/core.py", line 308, in send_recv
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
  File "<string>", line 3, in raise_exc_info
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1021, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/distributed/core.py", line 203, in read
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
  File "/home/jreback/miniconda2/envs/bmc3/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
  File "<string>", line 3, in raise_exc_info
tornado.iostream.StreamClosedError: Stream is closed