Parallel simulation failures on Linux

[ischoegl]

Parallel output fails on a Linux machine (running on 6 cores). See also updated commandline notebook ... curiously, the issue does not appear consistently: verbose output -v leads to more failures, and number of cores may play a role (it is not clear how many cores are used for testing pipelines on GitHub).

_________________________________________________________________ TestWrap.test_commandline __________________________________________________________________

self = <test_ctwrap.TestWrap testMethod=test_commandline>

    def test_commandline(self):
        cmd = 'ctwrap'
        name = self._module.__name__.split('.')[-1]
        yaml = "{}".format(Path(EXAMPLES) / self._yaml)
        pars = [name, yaml, '--parallel']
    
        self.maxDiff = None
        process = subprocess.Popen([cmd] + pars,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE)
        _, stderr = process.communicate()
>       self.assertEqual(stderr.decode(), '')
E       AssertionError: 'Process Process-1:\nTraceback (most recen[518 chars]nt\n' != ''
E       - Process Process-1:
E       - Traceback (most recent call last):
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E       -     self.run()
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E       -     self._target(*self._args, **self._kwargs)
E       -   File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 695, in worker
E       -     obj._save_metadata(metadata)
E       - UnboundLocalError: local variable 'obj' referenced before assignment

tests/test_ctwrap.py:85: AssertionError
_______________________________________________________________ TestIgnition.test_commandline ________________________________________________________________

self = <test_ctwrap.TestIgnition testMethod=test_commandline>

    def test_commandline(self):
        cmd = 'ctwrap'
        name = self._module.__name__.split('.')[-1]
        yaml = "{}".format(Path(EXAMPLES) / self._yaml)
        pars = [name, yaml, '--parallel']
    
        self.maxDiff = None
        process = subprocess.Popen([cmd] + pars,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE)
        _, stderr = process.communicate()
>       self.assertEqual(stderr.decode(), '')
E       AssertionError: 'Process Process-1:\nTraceback (most recen[3714 chars]')\n' != ''
E       - Process Process-1:
E       - Traceback (most recent call last):
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E       -     self.run()
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E       -     self._target(*self._args, **self._kwargs)
E       -   File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 687, in worker
E       -     obj._save(task=task)
E       -   File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 164, in _save
E       -     with h5py.File(filename, 'r') as hdf:
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 427, in __init__
E       -     swmr=swmr)
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 190, in make_fid
E       -     fid = h5f.open(name, flags, fapl=fapl)
E       -   File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
E       -   File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
E       -   File "h5py/h5f.pyx", line 96, in h5py.h5f.open
E       - OSError: Unable to open file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')
E       - Process Process-3:
E       - Traceback (most recent call last):
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E       -     self.run()
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E       -     self._target(*self._args, **self._kwargs)
E       -   File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 687, in worker
E       -     obj._save(task=task)
E       -   File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 164, in _save
E       -     with h5py.File(filename, 'r') as hdf:
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 427, in __init__
E       -     swmr=swmr)
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 190, in make_fid
E       -     fid = h5f.open(name, flags, fapl=fapl)
E       -   File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
E       -   File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
E       -   File "h5py/h5f.pyx", line 96, in h5py.h5f.open
E       - OSError: Unable to open file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')
E       - Process Process-2:
E       - Traceback (most recent call last):
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E       -     self.run()
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E       -     self._target(*self._args, **self._kwargs)
E       -   File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 687, in worker
E       -     obj._save(task=task)
E       -   File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 164, in _save
E       -     with h5py.File(filename, 'r') as hdf:
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 427, in __init__
E       -     swmr=swmr)
E       -   File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 190, in make_fid
E       -     fid = h5f.open(name, flags, fapl=fapl)
E       -   File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
E       -   File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
E       -   File "h5py/h5f.pyx", line 96, in h5py.h5f.open
E       - OSError: Unable to open file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')

tests/test_ctwrap.py:85: AssertionError

[ischoegl]

Closed via #33