You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
When using a distributed client for adaptive I get the following error:
Attached a notebook which reproduces the issue Issue with adaptive.txt
distributed.client - WARNING - Couldn't gather keys {'f_divergent_1d-e860261558c8b0627742c060e2601e1a': []}
tornado.application - ERROR - Exception in callback functools.partial(<function wrap..null_wrapper at 0x7fce980e1950>, <tornado.concurrent.Future object at 0x7fce98146240>)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
ret = callback()
File "/opt/conda/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
future.result()
File "/opt/conda/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info)
File "", line 4, in raise_exc_info
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
yielded = self.gen.throw(*exc_info)
File "/opt/conda/lib/python3.6/site-packages/distributed/cfexecutor.py", line 22, in _cascade_future
result = yield future._result(raiseit=False)
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
value = future.result()
File "/opt/conda/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info)
File "", line 4, in raise_exc_info
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
yielded = self.gen.throw(*exc_info)
File "/opt/conda/lib/python3.6/site-packages/distributed/client.py", line 194, in _result
result = yield self.client._gather([self])
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
value = future.result()
File "/opt/conda/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info)
File "", line 4, in raise_exc_info
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
yielded = self.gen.send(value)
File "/opt/conda/lib/python3.6/site-packages/distributed/client.py", line 1415, in _gather
self.futures[key].reset()
KeyError: 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
distributed.scheduler - ERROR - 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 1954, in handle_client
result = handler(**msg)
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 2838, in report_on_key
ts = self.tasks[key]
KeyError: 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
distributed.scheduler - ERROR - Exception in handle_client
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 1954, in handle_client
result = handler(**msg)
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 2838, in report_on_key
ts = self.tasks[key]
KeyError: 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
The text was updated successfully, but these errors were encountered:
I can't reliably reproduce this error. I just realized that shutdown_executor still defaults to True on the runners; maybe this error has something to do with the executor being shut down before all the data has come back?
When using a distributed client for adaptive I get the following error:
Attached a notebook which reproduces the issue
Issue with adaptive.txt
distributed.client - WARNING - Couldn't gather keys {'f_divergent_1d-e860261558c8b0627742c060e2601e1a': []}
tornado.application - ERROR - Exception in callback functools.partial(<function wrap..null_wrapper at 0x7fce980e1950>, <tornado.concurrent.Future object at 0x7fce98146240>)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py", line 605, in _run_callback
ret = callback()
File "/opt/conda/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
future.result()
File "/opt/conda/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info)
File "", line 4, in raise_exc_info
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
yielded = self.gen.throw(*exc_info)
File "/opt/conda/lib/python3.6/site-packages/distributed/cfexecutor.py", line 22, in _cascade_future
result = yield future._result(raiseit=False)
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
value = future.result()
File "/opt/conda/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info)
File "", line 4, in raise_exc_info
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
yielded = self.gen.throw(*exc_info)
File "/opt/conda/lib/python3.6/site-packages/distributed/client.py", line 194, in _result
result = yield self.client._gather([self])
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
value = future.result()
File "/opt/conda/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info)
File "", line 4, in raise_exc_info
File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 1069, in run
yielded = self.gen.send(value)
File "/opt/conda/lib/python3.6/site-packages/distributed/client.py", line 1415, in _gather
self.futures[key].reset()
KeyError: 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
distributed.scheduler - ERROR - 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 1954, in handle_client
result = handler(**msg)
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 2838, in report_on_key
ts = self.tasks[key]
KeyError: 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
distributed.scheduler - ERROR - Exception in handle_client
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 1954, in handle_client
result = handler(**msg)
File "/opt/conda/lib/python3.6/site-packages/distributed/scheduler.py", line 2838, in report_on_key
ts = self.tasks[key]
KeyError: 'f_divergent_1d-e860261558c8b0627742c060e2601e1a'
The text was updated successfully, but these errors were encountered: