Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 117 lines (107 sloc) 3.49 kB
57699f2 @chapmanb Use generalized map framework in automated workflow; allow missing pi…
authored
1 """Helpful utilities for building analysis pipelines.
d2ba583 @chapmanb Move general utilities out of Picard directory; add in helper functio…
authored
2 """
3 import os
4 import tempfile
5 import shutil
6 import contextlib
7 import itertools
8 import functools
9 import warnings
10
11 try:
12 import multiprocessing
13 from multiprocessing.pool import IMapIterator
14 except ImportError:
15 multiprocessing = None
16 try:
4bec268 @chapmanb Move general utilities out of Picard directory; add in helper functio…
authored
17 ipclient = None
18 #with warnings.catch_warnings():
19 # warnings.filterwarnings("ignore", category=DeprecationWarning)
20 # from IPython.kernel import client as ipclient
d2ba583 @chapmanb Move general utilities out of Picard directory; add in helper functio…
authored
21 except ImportError:
22 ipclient = None
23
24 @contextlib.contextmanager
25 def cpmap(cores=1, ipython=False):
26 """Configurable parallel map context manager.
27
28 Returns appropriate map compatible function based on configuration:
29 - Local single core (the default)
30 - Multiple local cores
31 - Parallelized on a cluster using ipython.
32 """
33 if ipython or cores=="ipython":
34 raise NotImplementedError("Not working yet")
35 if ipclient is None:
36 raise ImportError("ipython parallelization not available.")
37 mec = ipclient.MultiEngineClient()
38 # Would be ideal to have an imap style lazy map
39 yield mec.map
40 elif int(cores) == 1:
41 yield itertools.imap
42 else:
43 if multiprocessing is None:
44 raise ImportError("multiprocessing not available")
45 # Fix to allow keyboard interrupts in multiprocessing: https://gist.github.com/626518
46 def wrapper(func):
47 def wrap(self, timeout=None):
48 return func(self, timeout=timeout if timeout is not None else 1e100)
49 return wrap
50 IMapIterator.next = wrapper(IMapIterator.next)
51 pool = multiprocessing.Pool(int(cores))
52 yield pool.imap
57699f2 @chapmanb Use generalized map framework in automated workflow; allow missing pi…
authored
53 pool.terminate()
d2ba583 @chapmanb Move general utilities out of Picard directory; add in helper functio…
authored
54
55 def map_wrap(f):
56 """Wrap standard function to easily pass into 'map' processing.
57 """
58 @functools.wraps(f)
59 def wrapper(*args, **kwargs):
60 return apply(f, *args, **kwargs)
61 return wrapper
62
63 @contextlib.contextmanager
64 def curdir_tmpdir():
65 """Context manager to create and remove a temporary directory.
66 """
67 tmp_dir_base = os.path.join(os.getcwd(), "tmp")
68 if not os.path.exists(tmp_dir_base):
69 # we could get an error here if multiple processes are creating
70 # the directory at the same time. Grr, concurrency.
71 try:
72 os.makedirs(tmp_dir_base)
73 except OSError:
74 pass
75 tmp_dir = tempfile.mkdtemp(dir=tmp_dir_base)
76 if not os.path.exists(tmp_dir):
77 os.makedirs(tmp_dir)
78 try :
79 yield tmp_dir
80 finally :
81 shutil.rmtree(tmp_dir)
82
83 @contextlib.contextmanager
84 def chdir(new_dir):
85 """Context manager to temporarily change to a new directory.
86
87 http://lucentbeing.com/blog/context-managers-and-the-with-statement-in-python/
88 """
89 cur_dir = os.getcwd()
90 if not os.path.exists(new_dir):
91 os.makedirs(new_dir)
92 os.chdir(new_dir)
93 try :
94 yield
95 finally :
96 os.chdir(cur_dir)
97
98 @contextlib.contextmanager
99 def tmpfile(*args, **kwargs):
100 """Make a tempfile, safely cleaning up file descriptors on completion.
101 """
102 (fd, fname) = tempfile.mkstemp(*args, **kwargs)
103 try:
104 yield fname
105 finally:
106 os.close(fd)
107 if os.path.exists(fname):
108 os.remove(fname)
b4e36eb @chapmanb Utility to create working directorie for project
authored
109
110 def create_dirs(config, names=None):
111 if names is None:
112 names = config["dir"].keys()
113 for dname in names:
114 d = config["dir"][dname]
115 if not os.path.exists(d):
116 os.makedirs(d)
Something went wrong with that request. Please try again.