Crash threading swt2

[wtfuzz]

I'm trying to run concurrent swt2 transforms in threads, but it's crashing. It works with a ProcessPoolExecutor (which is obviously much slower, since we have to pickle the numpy data and fork, and pywt releases the GIL so threads should scale).

Test Case

import pywt
import numpy as np
import concurrent.futures as futures

class MWT(object):
  def __init__(self, wavelets, levels=2):
    self.wavelets = wavelets
    self.levels = levels
    self.wavelet_count = len(wavelets)

  def worker(self, x, wavelet):
    w = pywt.swt2(x, wavelet, level=self.levels)
    return w

  def transform(self, array):
    workers = []
    with futures.ThreadPoolExecutor() as executor:
      for w in self.wavelets:
        e = executor.submit(self.worker, array, w)
        workers.append(e)

    for w in workers:
      res = w.result()
      print(len(res))

def main():
  wavelets = [
    'db1',
    'db2',
    'db3',
    'db4',
    'db5'
  ]
  mwt = MWT(wavelets)

  array = np.zeros((256,256))
  mwt.transform(array)

  pass

if __name__ == '__main__':
  main()

Stack Trace

[Switching to Thread 0x7fffdca1c700 (LWP 5023)]
0x000000000048d9c9 in ?? ()
(gdb) bt
#0  0x000000000048d9c9 in ?? ()
#1  0x00007fffdf0407d9 in _unellipsify (__pyx_v_index=<optimized out>, __pyx_v_ndim=1) at pywt/_extensions/_swt.c:14928
#2  0x00007fffdf049833 in __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__ (__pyx_v_index=<optimized out>, __pyx_v_self=0x7fffdf7be048)
    at pywt/_extensions/_swt.c:11251
#3  __pyx_memoryview___getitem__ (__pyx_v_self=0x7fffdf7be048, __pyx_v_index=<optimized out>) at pywt/_extensions/_swt.c:11191
#4  0x00007fffdf0346ea in __pyx_sq_item_memoryview (o=0x7fffdf7be048, i=<optimized out>) at pywt/_extensions/_swt.c:20971
#5  0x00007ffff624d134 in ?? () from /home/fuzz/.local/lib/python3.6/site-packages/numpy/core/multiarray.cpython-36m-x86_64-linux-gnu.so
#6  0x00007ffff624d3cd in ?? () from /home/fuzz/.local/lib/python3.6/site-packages/numpy/core/multiarray.cpython-36m-x86_64-linux-gnu.so
#7  0x000000000056e02b in ?? ()
#8  0x0000000000570506 in ?? ()
#9  0x000000000057200e in _PyArg_ParseTupleAndKeywords_SizeT ()
#10 0x00007ffff62e9999 in ?? () from /home/fuzz/.local/lib/python3.6/site-packages/numpy/core/multiarray.cpython-36m-x86_64-linux-gnu.so
#11 0x00000000004c4763 in PyCFunction_Call ()
#12 0x00007fffdf057c7d in __Pyx_PyObject_Call (kw=0x7fffdedc90d8, arg=0x7fffdf2759b0, func=0x7ffff65e90d8) at pywt/_extensions/_swt.c:22674
#13 __pyx_f_4pywt_11_extensions_4_swt_swt_axis (__pyx_v_data=__pyx_v_data@entry=0x7fffdedc2ee0, __pyx_v_level=__pyx_v_level@entry=1, __pyx_v_start_level=<optimized out>, 
    __pyx_optional_args=__pyx_optional_args@entry=0x7fffdca1ab20, __pyx_skip_dispatch=0, __pyx_v_wavelet=<optimized out>) at pywt/_extensions/_swt.c:5287
#14 0x00007fffdf05bd7d in __pyx_pf_4pywt_11_extensions_4_swt_4swt_axis (__pyx_self=<optimized out>, __pyx_v_axis=1, __pyx_v_start_level=<optimized out>, __pyx_v_level=1, 
    __pyx_v_wavelet=0x7fffdf02bcc8, __pyx_v_data=0x7fffdedc2ee0) at pywt/_extensions/_swt.c:6072
#15 __pyx_pw_4pywt_11_extensions_4_swt_5swt_axis (__pyx_self=<optimized out>, __pyx_args=<optimized out>, __pyx_kwds=<optimized out>) at pywt/_extensions/_swt.c:6052
#16 0x00000000004c4b0b in _PyCFunction_FastCallKeywords ()
#17 0x000000000054f3c4 in ?? ()
#18 0x0000000000551ee0 in _PyEval_EvalFrameDefault ()
#19 0x000000000054efc1 in ?? ()
#20 0x000000000054f24d in ?? ()
#21 0x0000000000553aaf in _PyEval_EvalFrameDefault ()
#22 0x000000000054efc1 in ?? ()
#23 0x000000000054f24d in ?? ()
#24 0x0000000000551ee0 in _PyEval_EvalFrameDefault ()
#25 0x000000000054e4c8 in ?? ()
#26 0x00000000005582c2 in _PyFunction_FastCallDict ()
#27 0x0000000000459c11 in _PyObject_Call_Prepend ()
#28 0x000000000045969e in PyObject_Call ()
#29 0x0000000000552029 in _PyEval_EvalFrameDefault ()
#30 0x000000000054e4c8 in ?? ()
#31 0x000000000054f4f6 in ?? ()
#32 0x0000000000553aaf in _PyEval_EvalFrameDefault ()
#33 0x000000000054efc1 in ?? ()
#34 0x000000000054ffee in PyEval_EvalCodeEx ()
#35 0x000000000048b86d in ?? ()
#36 0x000000000045969e in PyObject_Call ()
#37 0x0000000000552029 in _PyEval_EvalFrameDefault ()
#38 0x000000000054e4c8 in ?? ()
#39 0x000000000054f4f6 in ?? ()
#40 0x0000000000553aaf in _PyEval_EvalFrameDefault ()
#41 0x000000000054e4c8 in ?? ()
#42 0x000000000054f4f6 in ?? ()
#43 0x0000000000553aaf in _PyEval_EvalFrameDefault ()
#44 0x000000000054e4c8 in ?? ()
#45 0x00000000005582c2 in _PyFunction_FastCallDict ()
#46 0x0000000000459c11 in _PyObject_Call_Prepend ()
#47 0x000000000045969e in PyObject_Call ()
#48 0x000000000058e2c2 in ?? ()
#49 0x00007ffff7bbd7fc in start_thread (arg=0x7fffdca1c700) at pthread_create.c:465
#50 0x00007ffff6d44b5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(gdb) 

[grlee77]

Thanks for reporting this. I also get a segmentation fault for your demo case. However, if I change swt2 to wavedec2 it works without problem.

Also, the basic concurrent swt2 test here passes:

pywt/pywt/tests/test_concurrent.py

Lines 46 to 64 in bf7be1f

	@dec.skipif(not futures_available)
	def test_concurrent_swt():
	# tests error-free concurrent operation (see gh-288)
	# swt on 1D data calls the Cython swt
	# other cases call swt_axes
	with warnings.catch_warnings():
	# can remove catch_warnings once the swt2 FutureWarning is removed
	warnings.simplefilter('ignore', FutureWarning)
	for swt_func, x in zip([pywt.swt, pywt.swt2, pywt.swtn],
	[np.ones(8), np.eye(16), np.eye(16)]):
	transform = partial(swt_func, wavelet='haar', level=1)
	for _ in range(10):
	arrs = [x.copy() for _ in range(100)]
	with futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
	results = list(ex.map(transform, arrs))
	# validate result from one of the concurrent runs
	expected_result = transform(x)
	_assert_all_coeffs_equal(expected_result, results[-1])

However, if I change level=1 to level=2 in the above test, the test also crashes, so that may be a hint on where to start looking.

[grlee77]

swt seems to work concurrently with level>1, but swt2 and swtn both experience the issue reported here.

[grlee77]

Actually, swt also fails if the data has more than one dimension, so the swt_axis Cython call seems to be the culprit.

[wtfuzz]

That would align with my stacktrace.

#15 __pyx_pw_4pywt_11_extensions_4_swt_5swt_axis (__pyx_self=<optimized out>, __pyx_args=<optimized out>, __pyx_kwds=<optimized out>) at pywt/_extensions/_swt.c:6052

[grlee77]

Can you also specify which python version you are using? (I see the segfault in Python 3.6, but it did not occur when I tried it with 2.7)

[wtfuzz]

$ python3 --version
Python 3.6.3

$ cat /etc/issue
Ubuntu 17.10

[wtfuzz]

Here's the backtrace from cygdb

(gdb) cy bt
#9  0x000000000054e4b5 in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a44428320>() at /usr/lib/python3.6/threading.py:884
       884                self._bootstrap_inner()
#14 0x000000000054e4b5 in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a44428f98>() at /usr/lib/python3.6/threading.py:916
       916                    self.run()
#19 0x000000000054e4b5 in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a4456e780>() at /usr/lib/python3.6/threading.py:864
       864                    self._target(*self._args, **self._kwargs)
#26 0x000000000054efac in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a42c34940>() at /usr/lib/python3.6/concurrent/futures/thread.py:69
        69                    work_item.run()
#31 0x000000000054e4b5 in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a42c34940>() at /usr/lib/python3.6/concurrent/futures/thread.py:56
        56                result = self.fn(*self.args, **self.kwargs)
#39 0x000000000054e4b5 in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a444d4390>() at mwt.py:23
        23          w = pywt.swt2(x, wavelet, level=self.levels)
#44 0x000000000054efac in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a4456e278>() at /home/fuzz/.local/lib/python3.6/site-packages/PyWavelets-1.0.0.dev0+bf7be1f-py3.6-linux-x86_64.egg/pywt/_swt.py:219
       219        coefs = swtn(data, wavelet, level, start_level, axes)
#49 0x000000000054efac in <Cython.Debugger.libpython.PyUnicodeObjectPtr object at 0x7f3a44428748>() at /home/fuzz/.local/lib/python3.6/site-packages/PyWavelets-1.0.0.dev0+bf7be1f-py3.6-linux-x86_64.egg/pywt/_swt.py:425
       425                                       axis=axis)[0]
#56 0x00007fffdedbd4b0 in swt_axis() at /home/fuzz/pywt/pywt/_extensions/_swt.pyx:184
       184            cD = np.empty(output_shape, dtype=data.dtype)
#67 0x00007fffdedd78f0 in __getitem__() at /home/fuzz/pywt/pywt/_extensions/_dwt.pyx:423
       423    cpdef downcoef(bint do_dec_a, cdata_t[::1] data, Wavelet wavelet, MODE mode, int level):
#69 0x00007fffdedb24f0 in _unellipsify() at /home/fuzz/pywt/pywt/_extensions/_dwt.pyx:991

[grlee77]

Sorry about the long delay. I looked at this again today and think I have tracked down the source of the segfault. It appears that the use of PyMem_Malloc here:

pywt/pywt/_extensions/c/common.h

Lines 38 to 41 in c39a9a2

	/* using Python's memory manager */
	#define wtmalloc(size) PyMem_Malloc(size)
	#define wtfree(ptr) PyMem_Free(ptr)
	void *wtcalloc(size_t, size_t);

requires the GIL, but we have used nogil in the Cython code to enable multithreading. The DWT code does not call wtcalloc, wtmalloc, or wtfree, so it does not experience the same problem.

If I switch this header to use the standard C malloc/free instead of the Python ones I no longer experience the segfault.

[grlee77]

closing as fixed by #367