From cdb98b335a17a4481d8a229f46c50eccbd745478 Mon Sep 17 00:00:00 2001 From: Andrei Cuceu Date: Sun, 23 Oct 2022 08:39:54 -0400 Subject: [PATCH] Make multiprocessing Pool safe in quickquasars The best practice for using multiprocessing.Pool is within a "with as" statement to safely terminate the pool. We (with Naim) found a bug when nproc > nfiles. This is the error message at the end of running quickquasars: Exception ignored in: Traceback (most recent call last): File "/global/homes/a/acuceu/.conda/envs/desienv/lib/python3.9/multiprocessing/pool.py", line 268, in del self._change_notifier.put(None) File "/global/homes/a/acuceu/.conda/envs/desienv/lib/python3.9/multiprocessing/queues.py", line 378, in put self._writer.send_bytes(obj) File "/global/homes/a/acuceu/.conda/envs/desienv/lib/python3.9/multiprocessing/connection.py", line 205, in send_bytes self._send_bytes(m[offset:offset + size]) File "/global/homes/a/acuceu/.conda/envs/desienv/lib/python3.9/multiprocessing/connection.py", line 416, in _send_bytes self._send(header + buf) File "/global/homes/a/acuceu/.conda/envs/desienv/lib/python3.9/multiprocessing/connection.py", line 373, in _send n = write(self._handle, buf) OSError: [Errno 9] Bad file descriptor This goes away when using "with multiprocessing.Pool(args.nproc) as pool:" to clean the pool after it's used. --- py/desisim/scripts/quickquasars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/desisim/scripts/quickquasars.py b/py/desisim/scripts/quickquasars.py index e0b490b1d..7eca57d2f 100644 --- a/py/desisim/scripts/quickquasars.py +++ b/py/desisim/scripts/quickquasars.py @@ -929,8 +929,8 @@ def main(args=None): "footprint_healpix_nside": footprint_healpix_nside , \ "bal":bal,"sfdmap":sfdmap,"eboss":eboss \ } for i,filename in enumerate(args.infile) ] - pool = multiprocessing.Pool(args.nproc) - pool.map(_func, func_args) + with multiprocessing.Pool(args.nproc) as pool: + pool.map(_func, func_args) else: for i,ifilename in enumerate(args.infile) : simulate_one_healpix(ifilename,args,model,obsconditions,