tests: crossbar fixture teardown wait on process sporadically fails #978
Description
Observed in a CI run: https://github.com/labgrid-project/labgrid/runs/7955580559?check_suite_focus=true#step:9:63
_______________ ERROR at teardown of test_place_match_duplicates _______________
self = PtyProcess.spawn(['/opt/hostedtoolcache/Python/3.9.13/x[64](https://github.com/labgrid-project/labgrid/runs/7955580559?check_suite_focus=true#step:9:65)/bin/crossbar', 'start', '--color', 'false', '--logformat', 'none'], cwd='/tmp/pytest-of-runner/pytest-0/test_place_match_duplicates0')
def isalive(self):
'''This tests if the child process is running or not. This is
non-blocking. If the child was terminated then this will read the
exitstatus or signalstatus of the child. This returns True if the child
process appears to be running or False if not. It can take literally
SECONDS for Solaris to return the right status. '''
if self.terminated:
return False
if self.flag_eof:
# This is for Linux, which requires the blocking form
# of waitpid to get the status of a defunct process.
# This is super-lame. The flag_eof would have been set
# in read_nonblocking(), so this should be safe.
waitpid_options = 0
else:
waitpid_options = os.WNOHANG
try:
> pid, status = os.waitpid(self.pid, waitpid_options)
E ChildProcessError: [Errno 10] No child processes
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/site-packages/ptyprocess/ptyprocess.py:711: ChildProcessError
During handling of the above exception, another exception occurred:
@contextmanager
def _wrap_ptyprocess_err():
"""Turn ptyprocess errors into our own ExceptionPexpect errors"""
try:
> yield
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/site-packages/pexpect/pty_spawn.py:23:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <pexpect.pty_spawn.spawn object at 0x7f8e0ecf8220>
def wait(self):
'''This waits until the child exits. This is a blocking call. This will
not read any data from the child, so this will block forever if the
child has unread output and has terminated. In other words, the child
may have printed output then called exit(), but, the child is
technically still alive until its output is read by the parent.
This method is non-blocking if :meth:`wait` has already been called
previously or :meth:`isalive` method returns False. It simply returns
the previously determined exit status.
'''
ptyproc = self.ptyproc
with _wrap_ptyprocess_err():
# exception may occur if "Is some other process attempting
# "job control with our child pid?"
> exitstatus = ptyproc.wait()
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/site-packages/pexpect/pty_spawn.py:688:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = PtyProcess.spawn(['/opt/hostedtoolcache/Python/3.9.13/x64/bin/crossbar', 'start', '--color', 'false', '--logformat', 'none'], cwd='/tmp/pytest-of-runner/pytest-0/test_place_match_duplicates0')
def wait(self):
'''This waits until the child exits. This is a blocking call. This will
not read any data from the child, so this will block forever if the
child has unread output and has terminated. In other words, the child
may have printed output then called exit(), but, the child is
technically still alive until its output is read by the parent. '''
> if self.isalive():
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/site-packages/ptyprocess/ptyprocess.py:[66](https://github.com/labgrid-project/labgrid/runs/7955580559?check_suite_focus=true#step:9:67)9:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = PtyProcess.spawn(['/opt/hostedtoolcache/Python/3.9.13/x64/bin/crossbar', 'start', '--color', 'false', '--logformat', 'none'], cwd='/tmp/pytest-of-runner/pytest-0/test_place_match_duplicates0')
def isalive(self):
'''This tests if the child process is running or not. This is
non-blocking. If the child was terminated then this will read the
exitstatus or signalstatus of the child. This returns True if the child
process appears to be running or False if not. It can take literally
SECONDS for Solaris to return the right status. '''
if self.terminated:
return False
if self.flag_eof:
# This is for Linux, which requires the blocking form
# of waitpid to get the status of a defunct process.
# This is super-lame. The flag_eof would have been set
# in read_nonblocking(), so this should be safe.
waitpid_options = 0
else:
waitpid_options = os.WNOHANG
try:
pid, status = os.waitpid(self.pid, waitpid_options)
except OSError as e:
# No child processes
if e.errno == errno.ECHILD:
> raise PtyProcessError('isalive() encountered condition ' +
'where "terminated" is 0, but there was no child ' +
'process. Did someone else call waitpid() ' +
'on our process?')
E ptyprocess.util.PtyProcessError: isalive() encountered condition where "terminated" is 0, but there was no child process. Did someone else call waitpid() on our process?
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/site-packages/ptyprocess/ptyprocess.py:715: PtyProcessError
During handling of the above exception, another exception occurred:
tmpdir = local('/tmp/pytest-of-runner/pytest-0/test_place_match_duplicates0')
crossbar_config = None
@pytest.fixture(scope='function')
def crossbar(tmpdir, crossbar_config):
if not find_spec('crossbar'):
pytest.skip("crossbar not found")
spawn = pexpect.spawn(
'crossbar start --color false --logformat none',
logfile=Prefixer(sys.stdout.buffer, 'crossbar'),
cwd=str(tmpdir))
try:
spawn.expect('Realm .* started')
spawn.expect('Guest .* started')
spawn.expect('Coordinator ready')
except:
print(f"crossbar startup failed with {spawn.before}")
raise
reader = threading.Thread(target=keep_reading, name='crossbar-reader', args=(spawn,), daemon=True)
reader.start()
yield spawn
# let coverage write its data:
# https://coverage.readthedocs.io/en/latest/subprocess.html#process-termination
print("stopping crossbar")
spawn.kill(SIGTERM)
spawn.expect(pexpect.EOF)
> spawn.wait()
tests/conftest.py:150:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/site-packages/pexpect/pty_spawn.py:[68](https://github.com/labgrid-project/labgrid/runs/7955580559?check_suite_focus=true#step:9:69)8: in wait
exitstatus = ptyproc.wait()
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/contextlib.py:137: in __exit__
self.gen.throw(typ, value, traceback)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
@contextmanager
def _wrap_ptyprocess_err():
"""Turn ptyprocess errors into our own ExceptionPexpect errors"""
try:
yield
except ptyprocess.PtyProcessError as e:
> raise ExceptionPexpect(*e.args)
E pexpect.exceptions.ExceptionPexpect: isalive() encountered condition where "terminated" is 0, but there was no child process. Did someone else call waitpid() on our process?
/opt/hostedtoolcache/Python/3.9.13/x64/lib/python3.9/site-packages/pexpect/pty_spawn.py:25: ExceptionPexpect
Probably introduced with #966.