Skip to content

Commit

Permalink
Added extra comments outlining not to share Pool processes between cl…
Browse files Browse the repository at this point in the history
…asses
  • Loading branch information
saeedamen committed Feb 17, 2017
1 parent abc7216 commit 55f3425
Showing 1 changed file with 25 additions and 3 deletions.
28 changes: 25 additions & 3 deletions findatapy/util/swimpool.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

class SwimPool(object):
"""Creating thread and process pools in a generic way. Allows users to specify the underlying thread or multiprocess library
they wish to use.
they wish to use. Note you can share Pool objects between processes.
"""

Expand All @@ -28,21 +28,43 @@ def __init__(self, multiprocessing_library = None):

self._multiprocessing_library = multiprocessing_library

def create_pool(self, thread_technique, thread_no, force_new = False):
if multiprocessing_library == 'multiprocess':
try:
import multiprocess;
multiprocess.freeze_support()
except:
pass
elif multiprocessing_library == 'multiprocessing_on_dill':
try:
import multiprocessing_on_dill;
multiprocessing_on_dill.freeze_support()
except:
pass
elif multiprocessing_library == 'multiprocessing':
try:
import multiprocessing;
multiprocessing.freeze_support()
except:
pass

def create_pool(self, thread_technique, thread_no, force_new = True):

if not(force_new) and self._pool is not None:
return self._pool

if thread_technique is "thread":
from multiprocessing.dummy import Pool
elif thread_technique is "multiprocessor":
elif thread_technique is "multiprocessing":
# most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing
# must use the multiprocessing_on_dill library otherwise can't pickle objects correctly
# note: currently not very stable
if self._multiprocessing_library == 'multiprocessing_on_dill':
from multiprocessing_on_dill import Pool
elif self._multiprocessing_library == 'multiprocess':
from multiprocess import Pool
elif self._multiprocessing_library == 'multiprocessing':
from multiprocessing import Pool

# from pathos.pools import ProcessPool as Pool

self._pool = Pool(thread_no)
Expand Down

0 comments on commit 55f3425

Please sign in to comment.