Skip to content

Commit

Permalink
TST, BUG: Re-raise MemoryError exception in test_large_zip's process (g…
Browse files Browse the repository at this point in the history
…h-16890)

Since #15893, test_large_zip's actual test is run in a child process,
so when this test raises a MemoryError exception, the exception is
lost and the @requires_memory decorator can't catch it to return
an xfail.

This commit uses a boolean variable in shared memory to flag if
the exception was raised, and in that case, re-raise it in the
parent process.

Fixes #16889
  • Loading branch information
antlarr-suse authored and charris committed Jul 19, 2020
1 parent 5bd4b2f commit 8399ab1
Showing 1 changed file with 22 additions and 8 deletions.
30 changes: 22 additions & 8 deletions numpy/lib/tests/test_io.py
Expand Up @@ -13,7 +13,8 @@
from io import BytesIO, StringIO
from datetime import datetime
import locale
from multiprocessing import Process
from multiprocessing import Process, Value
from ctypes import c_bool

import numpy as np
import numpy.ma as ma
Expand Down Expand Up @@ -574,16 +575,29 @@ def test_unicode_and_bytes_fmt(self, fmt, iotype):
@pytest.mark.slow
@requires_memory(free_bytes=7e9)
def test_large_zip(self):
def check_large_zip():
# The test takes at least 6GB of memory, writes a file larger than 4GB
test_data = np.asarray([np.random.rand(np.random.randint(50,100),4)
for i in range(800000)], dtype=object)
with tempdir() as tmpdir:
np.savez(os.path.join(tmpdir, 'test.npz'), test_data=test_data)
def check_large_zip(memoryerror_raised):
memoryerror_raised.value = False
try:
# The test takes at least 6GB of memory, writes a file larger
# than 4GB
test_data = np.asarray([np.random.rand(
np.random.randint(50,100),4)
for i in range(800000)], dtype=object)
with tempdir() as tmpdir:
np.savez(os.path.join(tmpdir, 'test.npz'),
test_data=test_data)
except MemoryError:
memoryerror_raised.value = True
raise
# run in a subprocess to ensure memory is released on PyPy, see gh-15775
p = Process(target=check_large_zip)
# Use an object in shared memory to re-raise the MemoryError exception
# in our process if needed, see gh-16889
memoryerror_raised = Value(c_bool)
p = Process(target=check_large_zip, args=(memoryerror_raised,))
p.start()
p.join()
if memoryerror_raised.value:
raise MemoryError("Child process raised a MemoryError exception")
assert p.exitcode == 0

class LoadTxtBase:
Expand Down

0 comments on commit 8399ab1

Please sign in to comment.