Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixup kernel death detection #998

Merged
merged 5 commits into from Apr 25, 2019
Merged
Changes from 3 commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -24,6 +24,8 @@
from .base import Preprocessor
from ..utils.exceptions import ConversionException

class KernelIsDead(RuntimeError):

This comment has been minimized.

Copy link
@MSeal

MSeal Apr 24, 2019

Collaborator

KernelDied or KernelDiedError

pass

class CellExecutionComplete(Exception):
"""
@@ -446,38 +448,48 @@ def _update_display_id(self, display_id, msg):
outputs[output_idx]['data'] = out['data']
outputs[output_idx]['metadata'] = out['metadata']


def _check_alive(self):
if not self.kc.is_alive():
self.log.error(
"Kernel died while waiting for execute reply.")
raise KernelIsDead("Kernel died")

def _wait_for_reply(self, msg_id, cell=None):
# wait for finish, with timeout
if self.timeout_func is not None and cell is not None:
timeout = self.timeout_func(cell)
else:
timeout = self.timeout

if not timeout or timeout < 0:
timeout = None
cummulative_time = 0
timeout_interval = 5
while True:
try:
if self.timeout_func is not None and cell is not None:
timeout = self.timeout_func(cell)
else:
timeout = self.timeout

if not timeout or timeout < 0:
timeout = None

if timeout is not None:
# timeout specified
msg = self.kc.shell_channel.get_msg(timeout=timeout)
else:
# no timeout specified, if kernel dies still handle this correctly
while True:
try:
# check every few seconds if kernel is still alive
msg = self.kc.shell_channel.get_msg(timeout=5)
except Empty:
# received no message, check if kernel is still alive
if not self.kc.is_alive():
self.log.error(
"Kernel died while waiting for execute reply.")
raise RuntimeError("Kernel died")
# kernel still alive, wait for a message
while True:
try:
msg = self.kc.shell_channel.get_msg(timeout=timeout_interval)
except Empty:
cummulative_time += timeout_interval
self._check_alive()
if timeout is None or cummulative_time <= timeout:
continue
# message received
break
else:
self.log.error( "Timeout waiting for execute reply (%is)." % self.timeout)
if self.interrupt_on_timeout:
self.log.error("Interrupting kernel")
self.km.interrupt_kernel()
break
else:
raise TimeoutError("Cell execution timed out")
break


except Empty:
self._check_alive()
self.log.error(
"Timeout waiting for execute reply (%is)." % self.timeout)
if self.interrupt_on_timeout:
@@ -20,7 +20,7 @@
import functools

from .base import PreprocessorTestsBase
from ..execute import ExecutePreprocessor, CellExecutionError, executenb
from ..execute import ExecutePreprocessor, CellExecutionError, executenb, KernelIsDead

import IPython
from mock import MagicMock
@@ -323,8 +323,29 @@ def timeout_func(source):
with pytest.raises(TimeoutError):
self.run_notebook(filename, dict(timeout_func=timeout_func), res)

def test_runtime_kernel_death(self):
"""Check that an error is raised when the kernel is_alive is false"""
filename = os.path.join(current_dir, 'files', 'Interrupt.ipynb')
with io.open(filename, 'r') as f:
input_nb = nbformat.read(f, 4)
res = self.build_resources()
res['metadata']['path'] = os.path.dirname(filename)

preprocessor = self.build_preprocessor({"timeout": 5})

try:
input_nb, output_nb = preprocessor(input_nb, {})
except TimeoutError as e:
pass
km, kc = preprocessor.start_new_kernel()

with patch.object(km, "is_alive") as alive_mock:
alive_mock.return_value = False
with pytest.raises(KernelIsDead):
input_nb, output_nb = preprocessor.preprocess(input_nb, {}, km=km)

@patch('jupyter_client.KernelManager.is_alive')
def test_kernel_death(self, alive_mock):
def test_startup_kernel_dead(self, alive_mock):
"""Check that an error is raised when the kernel is_alive is false"""
current_dir = os.path.dirname(__file__)
filename = os.path.join(current_dir, 'files', 'Interrupt.ipynb')
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.