Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

STY: PEP8 cleanup.

The only warnings left are for too-long lines in docstrings, and for no spaces
around the "**" operator.
  • Loading branch information...
commit 764d76153e6948df37795a643714e85765a11ca0 1 parent 05be47c
@rgommers rgommers authored
View
1  doc/__init__.py
@@ -2,4 +2,3 @@
This is a phony __init__.py file, so that nose finds the doctests in this
directory.
"""
-
View
5 doc/conf.py
@@ -11,7 +11,8 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
-import sys, os
+import sys
+import os
import joblib
@@ -28,7 +29,7 @@
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath', 'numpydoc',
- 'phantom_import', 'autosummary',
+ 'phantom_import', 'autosummary',
'sphinx.ext.coverage']
#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest']
View
19 joblib/__init__.py
@@ -13,15 +13,15 @@
**BSD-licensed**.
- ============================== ==============================================
+ ============================== ============================================
**User documentation**: http://packages.python.org/joblib
-
+
**Download packages**: http://pypi.python.org/pypi/joblib#downloads
-
+
**Source code**: http://github.com/joblib/joblib
**Report issues**: http://github.com/joblib/joblib/issues
- ============================== ==============================================
+ ============================== ============================================
Vision
@@ -36,7 +36,7 @@
over, for instance when prototyping computational-heavy jobs (as in
scientific development), but hand-crafted solution to aleviate this
issue is error-prone and often leads to unreproducible results
-
+
* **Persist to disk transparently**: persisting in an efficient way
arbitrary objects containing large data is hard. In addition,
hand-written persistence does not link easily the file on disk to the
@@ -46,7 +46,7 @@
It strives to address these problems while **leaving your code and your
flow control as unmodified as possible** (no framework, no new
-paradigms).
+paradigms).
Main features
------------------
@@ -75,7 +75,7 @@
>>> c = square(a)
>>> # The above call did not trigger an evaluation
-2) **Embarrassingly parallel helper:** to make is easy to write readable
+2) **Embarrassingly parallel helper:** to make is easy to write readable
parallel code and debug it quickly:
>>> from joblib import Parallel, delayed
@@ -88,10 +88,10 @@
progressively acquire better logging mechanism to help track what
has been ran, and capture I/O easily. In addition, Joblib will
provide a few I/O primitives, to easily define define logging and
- display streams, and provide a way of compiling a report.
+ display streams, and provide a way of compiling a report.
We want to be able to quickly inspect what has been run.
-..
+..
>>> import shutil ; shutil.rmtree('/tmp/joblib/')
"""
@@ -104,4 +104,3 @@
from .hashing import hash
from .numpy_pickle import dump, load
from .parallel import Parallel, delayed, cpu_count
-
View
14 joblib/disk.py
@@ -2,7 +2,7 @@
Disk management utilities.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2010 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -12,9 +12,9 @@
import shutil
import time
+
def disk_used(path):
- """ Return the disk usage in a directory.
- """
+ """ Return the disk usage in a directory."""
size = 0
for file in os.listdir(path) + ['.']:
stat = os.stat(os.path.join(path, file))
@@ -23,7 +23,7 @@ def disk_used(path):
else:
# on some platform st_blocks is not available (e.g., Windows)
# approximate by rounding to next multiple of 512
- size += (stat.st_size // 512 + 1) * 512;
+ size += (stat.st_size // 512 + 1) * 512
# We need to convert to int to avoid having longs on some systems (we
# don't want longs to avoid problems we SQLite)
return int(size / 1024.)
@@ -47,9 +47,10 @@ def memstr_to_kbytes(text):
# then retry once. if it still fails, raise the exception
RM_SUBDIRS_RETRY_TIME = 0.1
+
def rm_subdirs(path, onerror=None):
"""Remove all subdirectories in this path.
-
+
The directory indicated by `path` is left in place, and its subdirectories
are erased.
@@ -86,6 +87,7 @@ def rm_subdirs(path, onerror=None):
shutil.rmtree(fullname, False, None)
break
except os.error, err:
- if err_count > 0: raise
+ if err_count > 0:
+ raise
err_count += 1
time.sleep(RM_SUBDIRS_RETRY_TIME)
View
106 joblib/format_stack.py
@@ -31,9 +31,11 @@
import traceback
import types
-INDENT = ' '*8
-################################################################################
+INDENT = ' ' * 8
+
+
+###############################################################################
# some internal-use functions
def safe_repr(value):
"""Hopefully pretty robust repr equivalent."""
@@ -63,11 +65,12 @@ def safe_repr(value):
except:
return 'UNRECOVERABLE REPR FAILURE'
-def eq_repr(value, repr=safe_repr):
+
+def eq_repr(value, repr=safe_repr):
return '=%s' % repr(value)
-################################################################################
+###############################################################################
def uniq_stable(elems):
"""uniq_stable(elems) -> list
@@ -89,10 +92,10 @@ def uniq_stable(elems):
return unique
-################################################################################
+###############################################################################
def fix_frame_records_filenames(records):
"""Try to fix the filenames in each record from inspect.getinnerframes().
-
+
Particularly, modules loaded from within zip files have useless filenames
attached to their code object, and inspect.getinnerframes() just uses it.
"""
@@ -106,14 +109,15 @@ def fix_frame_records_filenames(records):
# __file__. It might also be None if the error occurred during
# import.
filename = better_fn
- fixed_records.append((frame, filename, line_no, func_name, lines, index))
+ fixed_records.append((frame, filename, line_no, func_name, lines,
+ index))
return fixed_records
def _fixed_getframes(etb, context=1, tb_offset=0):
- LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
+ LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
- records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
+ records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
# If the error is at the console, don't build any context, since it would
# otherwise produce 5 blank lines printed out (there is no file at the
@@ -128,10 +132,10 @@ def _fixed_getframes(etb, context=1, tb_offset=0):
aux = traceback.extract_tb(etb)
assert len(records) == len(aux)
- for i, (file, lnum, _, _) in enumerate(aux):
- maybeStart = lnum-1 - context//2
- start = max(maybeStart, 0)
- end = start + context
+ for i, (file, lnum, _, _) in enumerate(aux):
+ maybeStart = lnum - 1 - context // 2
+ start = max(maybeStart, 0)
+ end = start + context
lines = linecache.getlines(file)[start:end]
# pad with empty lines if necessary
if maybeStart < 0:
@@ -156,17 +160,17 @@ def _format_traceback_lines(lnum, index, lines, lvals=None):
# This is the line with the error
pad = numbers_width - len(str(i))
if pad >= 3:
- marker = '-'*(pad-3) + '-> '
+ marker = '-' * (pad - 3) + '-> '
elif pad == 2:
- marker = '> '
+ marker = '> '
elif pad == 1:
marker = '>'
else:
marker = ''
num = marker + str(i)
else:
- num = '%*s' % (numbers_width,i)
- line = '%s %s' %(num, line)
+ num = '%*s' % (numbers_width, i)
+ line = '%s %s' % (num, line)
res.append(line)
if lvals and i == lnum:
@@ -175,7 +179,7 @@ def _format_traceback_lines(lnum, index, lines, lvals=None):
return res
-def format_records(records): #, print_globals=False):
+def format_records(records): # , print_globals=False):
# Loop over all records printing context and info
frames = []
abspath = os.path.abspath
@@ -196,15 +200,15 @@ def format_records(records): #, print_globals=False):
# able to remove this try/except when 2.4 becomes a
# requirement. Bug details at http://python.org/sf/1005466
print "\nJoblib's exception reporting continues...\n"
-
+
if func == '?':
call = ''
else:
# Decide whether to include variable details or not
try:
- call = 'in %s%s' % (func,inspect.formatargvalues(args,
- varargs, varkw,
- locals, formatvalue=eq_repr))
+ call = 'in %s%s' % (func, inspect.formatargvalues(args,
+ varargs, varkw, locals,
+ formatvalue=eq_repr))
except KeyError:
# Very odd crash from inspect.formatargvalues(). The
# scenario under which it appeared was a call to
@@ -228,7 +232,7 @@ def tokeneater(token_type, token, start, end, line):
there is no way to disambguate partial dotted structures until
the full list is known. The caller is responsible for pruning
the final list of duplicates before using it."""
-
+
# build composite names
if token == '.':
try:
@@ -271,28 +275,28 @@ def linereader(file=file, lnum=[lnum], getline=linecache.getline):
except IndexError:
# signals exit of tokenizer
pass
- except tokenize.TokenError,msg:
+ except tokenize.TokenError, msg:
print ("An unexpected error occurred while tokenizing input\n"
"The following traceback may be corrupted or invalid\n"
"The error message is: %s\n" % msg)
-
+
# prune names list of duplicates, but keep the right order
unique_names = uniq_stable(names)
# Start loop over vars
lvals = []
for name_full in unique_names:
- name_base = name_full.split('.',1)[0]
+ name_base = name_full.split('.', 1)[0]
if name_base in frame.f_code.co_varnames:
- if locals.has_key(name_base):
+ if name_base in locals.keys():
try:
- value = repr(eval(name_full,locals))
+ value = repr(eval(name_full, locals))
except:
value = "undefined"
else:
value = "undefined"
name = name_full
- lvals.append('%s = %s' % (name,value))
+ lvals.append('%s = %s' % (name, value))
#elif print_globals:
# if frame.f_globals.has_key(name_base):
# try:
@@ -308,21 +312,21 @@ def linereader(file=file, lnum=[lnum], getline=linecache.getline):
else:
lvals = ''
- level = '%s\n%s %s\n' % (75*'.', link, call)
+ level = '%s\n%s %s\n' % (75 * '.', link, call)
if index is None:
frames.append(level)
else:
- frames.append('%s%s' % (level,''.join(
+ frames.append('%s%s' % (level, ''.join(
_format_traceback_lines(lnum, index, lines, lvals))))
-
+
return frames
-################################################################################
+###############################################################################
def format_exc(etype, evalue, etb, context=5, tb_offset=0):
""" Return a nice text document describing the traceback.
-
+
Parameters
-----------
etype, evalue, etb: as returned by sys.exc_info
@@ -340,9 +344,9 @@ def format_exc(etype, evalue, etb, context=5, tb_offset=0):
pyver = 'Python ' + string.split(sys.version)[0] + ': ' + sys.executable
date = time.ctime(time.time())
pid = 'PID: %i' % os.getpid()
-
- head = '%s%s%s\n%s%s%s' % (etype, ' '*(75-len(str(etype))-len(date)),
- date, pid, ' '*(75-len(str(pid))-len(pyver)),
+
+ head = '%s%s%s\n%s%s%s' % (etype, ' ' * (75 - len(str(etype)) - len(date)),
+ date, pid, ' ' * (75 - len(str(pid)) - len(pyver)),
pyver)
# Flush cache before calling inspect. This helps alleviate some of the
@@ -353,16 +357,17 @@ def format_exc(etype, evalue, etb, context=5, tb_offset=0):
records = _fixed_getframes(etb, context, tb_offset)
except:
raise
- print '\nUnfortunately, your original traceback can not be constructed.\n'
+ print '\nUnfortunately, your original traceback can not be ' + \
+ 'constructed.\n'
return ''
# Get (safely) a string form of the exception info
try:
- etype_str,evalue_str = map(str,(etype,evalue))
+ etype_str, evalue_str = map(str, (etype, evalue))
except:
# User exception is improperly defined.
- etype,evalue = str,sys.exc_info()[:2]
- etype_str,evalue_str = map(str,(etype,evalue))
+ etype, evalue = str, sys.exc_info()[:2]
+ etype_str, evalue_str = map(str, (etype, evalue))
# ... and format it
exception = ['%s: %s' % (etype_str, evalue_str)]
if type(evalue) is types.InstanceType:
@@ -375,7 +380,7 @@ def format_exc(etype, evalue, etb, context=5, tb_offset=0):
exception.append(
'Exception reporting error (object with broken dir()):'
)
- etype_str, evalue_str = map(str,sys.exc_info()[:2])
+ etype_str, evalue_str = map(str, sys.exc_info()[:2])
exception.append('%s: %s' % (etype_str, evalue_str))
names = []
for name in names:
@@ -383,13 +388,13 @@ def format_exc(etype, evalue, etb, context=5, tb_offset=0):
exception.append('\n%s%s = %s' % (INDENT, name, value))
frames = format_records(records)
- return '%s\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )
+ return '%s\n%s\n%s' % (head, '\n'.join(frames), ''.join(exception[0]))
-################################################################################
+###############################################################################
def format_outer_frames(context=5, stack_start=None, stack_end=None,
ignore_ipython=True):
- LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
+ LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
records = inspect.getouterframes(inspect.currentframe())
output = list()
@@ -407,12 +412,12 @@ def format_outer_frames(context=5, stack_start=None, stack_end=None,
filename = filename[:-4] + '.py'
if ignore_ipython:
# Hack to avoid printing the interals of IPython
- if (os.path.basename(filename) == 'iplib.py'
+ if (os.path.basename(filename) == 'iplib.py'
and func_name in ('safe_execfile', 'runcode')):
break
- maybeStart = line_no -1 - context//2
- start = max(maybeStart, 0)
- end = start + context
+ maybeStart = line_no - 1 - context // 2
+ start = max(maybeStart, 0)
+ end = start + context
lines = linecache.getlines(filename)[start:end]
# pad with empty lines if necessary
if maybeStart < 0:
@@ -425,6 +430,3 @@ def format_outer_frames(context=5, stack_start=None, stack_end=None,
buf[LINES_POS] = lines
output.append(tuple(buf))
return '\n'.join(format_records(output[stack_end:stack_start:-1]))
-
-
-
View
38 joblib/func_inspect.py
@@ -2,7 +2,7 @@
My own variation on function-specific inspect-like features.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -11,9 +11,10 @@
import warnings
import os
+
def get_func_code(func):
""" Attempts to retrieve a reliable function code hash.
-
+
The reason we don't use inspect.getsource is that it caches the
source, whereas we want this to be modified on the fly when the
function is modified.
@@ -39,7 +40,8 @@ def get_func_code(func):
source_file_obj = file(source_file)
first_line = func.func_code.co_firstlineno
# All the lines after the function definition:
- source_lines = list(itertools.islice(source_file_obj, first_line-1, None))
+ source_lines = list(itertools.islice(source_file_obj, first_line - 1,
+ None))
return ''.join(inspect.getblock(source_lines)), source_file, first_line
except:
# If the source code fails, we use the hash. This is fragile and
@@ -54,7 +56,7 @@ def get_func_code(func):
return repr(func), source_file, -1
-def get_func_name(func, resolv_alias=True, win_characters=True):
+def get_func_name(func, resolv_alias=True, win_characters=True):
""" Return the function import path (as a list of module names), and
a name for the function.
@@ -94,7 +96,7 @@ def get_func_name(func, resolv_alias=True, win_characters=True):
module = module.split('.')
if hasattr(func, 'func_name'):
name = func.func_name
- elif hasattr(func, '__name__'):
+ elif hasattr(func, '__name__'):
name = func.__name__
else:
name = 'unknown'
@@ -105,7 +107,7 @@ def get_func_name(func, resolv_alias=True, win_characters=True):
if not func.func_globals[name] is func:
name = '%s-alias' % name
if inspect.ismethod(func):
- # We need to add the name of the class
+ # We need to add the name of the class
if hasattr(func, 'im_class'):
klass = func.im_class
module.append(klass.__name__)
@@ -126,7 +128,7 @@ def filter_args(func, ignore_lst, *args, **kwargs):
func: callable
Function giving the argument specification
ignore_lst: list of strings
- List of arguments to ignore (either a name of an argument
+ List of arguments to ignore (either a name of an argument
in the function spec, or '*', or '**')
*args: list
Positional arguments passed to the function.
@@ -146,18 +148,18 @@ def filter_args(func, ignore_lst, *args, **kwargs):
raise ValueError('ignore_lst must be a list of parameters to ignore '
'%s (type %s) was given' % (ignore_lst, type(ignore_lst)))
# Special case for functools.partial objects
- if (not inspect.ismethod(func) and not inspect.isfunction(func)):
+ if (not inspect.ismethod(func) and not inspect.isfunction(func)):
if ignore_lst:
warnings.warn('Cannot inspect object %s, ignore list will '
'not work.' % func, stacklevel=2)
- return {'*':args, '**':kwargs}
+ return {'*': args, '**': kwargs}
arg_spec = inspect.getargspec(func)
# We need to if/them to account for different versions of Python
if hasattr(arg_spec, 'args'):
- arg_names = arg_spec.args
+ arg_names = arg_spec.args
arg_defaults = arg_spec.defaults
arg_keywords = arg_spec.keywords
- arg_varargs = arg_spec.varargs
+ arg_varargs = arg_spec.varargs
else:
arg_names, arg_varargs, arg_keywords, arg_defaults = arg_spec
arg_defaults = arg_defaults or {}
@@ -165,7 +167,7 @@ def filter_args(func, ignore_lst, *args, **kwargs):
# First argument is 'self', it has been removed by Python
# we need to add it back:
args = [func.im_self, ] + args
- # XXX: Maybe I need an inspect.isbuiltin to detect C-level methods, such
+ # XXX: Maybe I need an inspect.isbuiltin to detect C-level methods, such
# as on ndarrays.
_, name = get_func_name(func, resolv_alias=False)
@@ -186,16 +188,15 @@ def filter_args(func, ignore_lst, *args, **kwargs):
# Missing argument
raise ValueError('Wrong number of arguments for %s%s:\n'
' %s(%s, %s) was called.'
- % (name,
+ % (name,
inspect.formatargspec(*inspect.getargspec(func)),
name,
repr(args)[1:-1],
- ', '.join('%s=%s' % (k, v)
+ ', '.join('%s=%s' % (k, v)
for k, v in kwargs.iteritems())
)
)
-
varkwargs = dict()
for arg_name, arg_value in kwargs.iteritems():
if arg_name in arg_dict:
@@ -209,7 +210,7 @@ def filter_args(func, ignore_lst, *args, **kwargs):
if arg_keywords is not None:
arg_dict['**'] = varkwargs
if arg_varargs is not None:
- varargs = args[arg_position+1:]
+ varargs = args[arg_position + 1:]
arg_dict['*'] = varargs
# Now remove the arguments to be ignored
@@ -218,7 +219,7 @@ def filter_args(func, ignore_lst, *args, **kwargs):
arg_dict.pop(item)
else:
raise ValueError("Ignore list: argument '%s' is not defined for "
- "function %s%s" %
+ "function %s%s" %
(item, name,
inspect.formatargspec(arg_names,
arg_varargs,
@@ -226,5 +227,4 @@ def filter_args(func, ignore_lst, *args, **kwargs):
arg_defaults,
)))
# XXX: Return a sorted list of pairs?
- return arg_dict
-
+ return arg_dict
View
17 joblib/hashing.py
@@ -1,9 +1,9 @@
"""
-Fast cryptographic hash of Python objects, with a special case for fast
+Fast cryptographic hash of Python objects, with a special case for fast
hashing of numpy arrays.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -13,6 +13,7 @@
import cStringIO
import types
+
class Hasher(pickle.Pickler):
""" A subclass of pickler, to do cryptographic hashing, rather than
pickling.
@@ -41,6 +42,7 @@ def save(self, obj):
obj = (func_name, inst, cls)
pickle.Pickler.save(self, obj)
+
class NumpyHasher(Hasher):
""" Special case the hasher for when numpy is loaded.
"""
@@ -76,7 +78,7 @@ def save(self, obj):
# XXX: There might be a more efficient way of doing this
self._hash.update(self.np.getbuffer(obj.flatten()))
- # We store the class, to be able to distinguish between
+ # We store the class, to be able to distinguish between
# Objects with the same binary content, but different
# classes.
if self.coerce_mmap and isinstance(obj, self.np.memmap):
@@ -86,7 +88,7 @@ def save(self, obj):
klass = self.np.ndarray
else:
klass = obj.__class__
- # We also return the dtype and the shape, to distinguish
+ # We also return the dtype and the shape, to distinguish
# different views on the same data with different dtypes.
# The object will be pickled by the pickler hashed at the end.
@@ -95,14 +97,14 @@ def save(self, obj):
def hash(obj, hash_name='md5', coerce_mmap=False):
- """ Quick calculation of a hash to identify uniquely Python objects
+ """ Quick calculation of a hash to identify uniquely Python objects
containing numpy arrays.
-
+
Parameters
-----------
hash_name: 'md5' or 'sha1'
- Hashing algorithm used. sha1 is supposedly safer, but md5 is
+ Hashing algorithm used. sha1 is supposedly safer, but md5 is
faster.
coerce_mmap: boolean
Make no difference between np.memmap and np.ndarray
@@ -112,4 +114,3 @@ def hash(obj, hash_name='md5', coerce_mmap=False):
else:
hasher = Hasher(hash_name=hash_name)
return hasher.hash(obj)
-
View
37 joblib/logger.py
@@ -4,7 +4,7 @@
This module needs much love to become useful.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2008 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -16,6 +16,7 @@
import logging
import pprint
+
def _squeeze_time(t):
"""Remove .1s to the time under Windows: this is the time it take to
stat files. This is needed to make results similar to timings under
@@ -26,24 +27,27 @@ def _squeeze_time(t):
else:
return t
+
def format_time(t):
t = _squeeze_time(t)
- return "%.1fs, %.1fmin" % (t, t/60.)
+ return "%.1fs, %.1fmin" % (t, t / 60.)
+
def short_format_time(t):
t = _squeeze_time(t)
if t > 60:
- return "%4.1fmin" % (t/60.)
+ return "%4.1fmin" % (t / 60.)
else:
return " %5.1fs" % (t)
-################################################################################
+
+###############################################################################
# class `Logger`
-################################################################################
+###############################################################################
class Logger(object):
""" Base class for logging messages.
"""
-
+
def __init__(self, depth=3):
"""
Parameters
@@ -75,9 +79,9 @@ def format(self, obj, indent=0):
return out
-################################################################################
+###############################################################################
# class `PrintTime`
-################################################################################
+###############################################################################
class PrintTime(object):
""" Print and log messages while keeping track of time.
"""
@@ -97,16 +101,16 @@ def __init__(self, logfile=None, logdir=None):
if os.path.exists(logfile):
# Rotate the logs
for i in range(1, 9):
- if os.path.exists(logfile+'.%i' % i):
+ if os.path.exists(logfile + '.%i' % i):
try:
- shutil.move(logfile+'.%i' % i,
- logfile+'.%i' % (i+1))
+ shutil.move(logfile + '.%i' % i,
+ logfile + '.%i' % (i + 1))
except:
"No reason failing here"
# Use a copy rather than a move, so that a process
# monitoring this file does not get lost.
try:
- shutil.copy(logfile, logfile+'.1')
+ shutil.copy(logfile, logfile + '.1')
except:
"No reason failing here"
try:
@@ -121,18 +125,18 @@ def __init__(self, logfile=None, logdir=None):
# XXX: We actually need a debug flag to disable this
# silent failure.
-
def __call__(self, msg='', total=False):
""" Print the time elapsed between the last call and the current
call, with an optional message.
"""
if not total:
time_lapse = time.time() - self.last_time
- full_msg = "%s: %s" % (msg, format_time(time_lapse) )
+ full_msg = "%s: %s" % (msg, format_time(time_lapse))
else:
# FIXME: Too much logic duplicated
time_lapse = time.time() - self.start_time
- full_msg = "%s: %.2fs, %.1f min" % (msg, time_lapse, time_lapse/60)
+ full_msg = "%s: %.2fs, %.1f min" % (msg, time_lapse,
+ time_lapse / 60)
print >> sys.stderr, full_msg
if self.logfile is not None:
try:
@@ -145,6 +149,3 @@ def __call__(self, msg='', total=False):
# XXX: We actually need a debug flag to disable this
# silent failure.
self.last_time = time.time()
-
-
-
View
108 joblib/memory.py
@@ -4,7 +4,7 @@
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -45,10 +45,10 @@
# object, and the interface to persist and query should be separated in
# the data store.
#
-# This would enable creating 'Memory' objects with a different logic for
+# This would enable creating 'Memory' objects with a different logic for
# pickling that would simply span a MemorizedFunc with the same
# store (or do we want to copy it to avoid cross-talks?), for instance to
-# implement HDF5 pickling.
+# implement HDF5 pickling.
# TODO: Same remark for the logger, and probably use the Python logging
# mechanism.
@@ -72,13 +72,13 @@ class JobLibCollisionWarning(UserWarning):
"""
-################################################################################
+###############################################################################
# class `MemorizedFunc`
-################################################################################
+###############################################################################
class MemorizedFunc(Logger):
- """ Callable object decorating a function for caching its return value
+ """ Callable object decorating a function for caching its return value
each time it is called.
-
+
All values are cached on the filesystem, in a deep directory
structure. Methods are provided to inspect the cache or clean it.
@@ -97,14 +97,14 @@ class MemorizedFunc(Logger):
arguments. Only used if save_npy was true when the
cache was created.
verbose: int, optional
- The verbosity flag, controls messages that are issued as
+ The verbosity flag, controls messages that are issued as
the function is revaluated.
"""
#-------------------------------------------------------------------------
# Public interface
#-------------------------------------------------------------------------
-
- def __init__(self, func, cachedir, ignore=None, save_npy=True,
+
+ def __init__(self, func, cachedir, ignore=None, save_npy=True,
mmap_mode=None, verbose=1, timestamp=None):
"""
Parameters
@@ -124,7 +124,7 @@ def __init__(self, func, cachedir, ignore=None, save_npy=True,
arguments. Only used if save_npy was true when the
cache was created.
verbose: int, optional
- Verbosity flag, controls the debug messages that are issued
+ Verbosity flag, controls the debug messages that are issued
as functions are revaluated. The higher, the more verbose
timestamp: float, optional
The reference time from which times in tracing messages
@@ -156,13 +156,12 @@ def __init__(self, func, cachedir, ignore=None, save_npy=True,
doc = func.__doc__
self.__doc__ = 'Memoized version of %s' % doc
-
def __call__(self, *args, **kwargs):
# Compare the function code with the previous to see if the
# function code has changed
output_dir, _ = self.get_output_dir(*args, **kwargs)
# FIXME: The statements below should be try/excepted
- if not (self._check_previous_func_code(stacklevel=3) and
+ if not (self._check_previous_func_code(stacklevel=3) and
os.path.exists(output_dir)):
return self.call(*args, **kwargs)
else:
@@ -173,32 +172,29 @@ def __call__(self, *args, **kwargs):
t = time.time() - t0
_, name = get_func_name(self.func)
msg = '%s cache loaded - %s' % (name, format_time(t))
- print max(0, (80 - len(msg)))*'_' + msg
+ print max(0, (80 - len(msg))) * '_' + msg
return out
except Exception:
# XXX: Should use an exception logger
- self.warn(
- 'Exception while loading results for '
- '(args=%s, kwargs=%s)\n %s' %
- (args, kwargs, traceback.format_exc())
- )
+ self.warn('Exception while loading results for '
+ '(args=%s, kwargs=%s)\n %s' %
+ (args, kwargs, traceback.format_exc()))
shutil.rmtree(output_dir, ignore_errors=True)
return self.call(*args, **kwargs)
-
def __reduce__(self):
""" We don't store the timestamp when pickling, to avoid the hash
depending from it.
In addition, when unpickling, we run the __init__
"""
- return (self.__class__, (self.func, self.cachedir, self.ignore,
+ return (self.__class__, (self.func, self.cachedir, self.ignore,
self.save_npy, self.mmap_mode, self._verbose))
#-------------------------------------------------------------------------
# Private interface
#-------------------------------------------------------------------------
-
+
def _get_func_dir(self, mkdir=True):
""" Get the directory corresponding to the cache for the
function.
@@ -210,13 +206,12 @@ def _get_func_dir(self, mkdir=True):
try:
os.makedirs(func_dir)
except OSError:
- """ Dir exists: we have a race condition here, when using
+ """ Dir exists: we have a race condition here, when using
multiprocessing.
"""
# XXX: Ugly
return func_dir
-
def get_output_dir(self, *args, **kwargs):
""" Returns the directory in which are persisted the results
of the function corresponding to the given arguments.
@@ -225,12 +220,11 @@ def get_output_dir(self, *args, **kwargs):
"""
coerce_mmap = (self.mmap_mode is not None)
argument_hash = hash(filter_args(self.func, self.ignore,
- *args, **kwargs),
+ *args, **kwargs),
coerce_mmap=coerce_mmap)
output_dir = os.path.join(self._get_func_dir(self.func),
- argument_hash)
+ argument_hash)
return output_dir, argument_hash
-
def _write_func_code(self, filename, func_code, first_line):
""" Write the function code and the filename to a file.
@@ -238,9 +232,8 @@ def _write_func_code(self, filename, func_code, first_line):
func_code = '%s %i\n%s' % (FIRST_LINE_TEXT, first_line, func_code)
file(filename, 'w').write(func_code)
-
def _check_previous_func_code(self, stacklevel=2):
- """
+ """
stacklevel is the depth a which this function is called, to
issue useful warnings to the user.
"""
@@ -252,7 +245,7 @@ def _check_previous_func_code(self, stacklevel=2):
func_code_file = os.path.join(func_dir, 'func_code.py')
try:
- if not os.path.exists(func_code_file):
+ if not os.path.exists(func_code_file):
raise IOError
old_func_code, old_first_line = \
extract_first_line(file(func_code_file).read())
@@ -263,14 +256,14 @@ def _check_previous_func_code(self, stacklevel=2):
return True
# We have differing code, is this because we are refering to
- # differing functions, or because the function we are refering as
+ # differing functions, or because the function we are refering as
# changed?
if old_first_line == first_line == -1:
_, func_name = get_func_name(self.func, resolv_alias=False,
win_characters=False)
if not first_line == -1:
- func_description = '%s (%s:%i)' % (func_name,
+ func_description = '%s (%s:%i)' % (func_name,
source_file, first_line)
else:
func_description = func_name
@@ -282,19 +275,19 @@ def _check_previous_func_code(self, stacklevel=2):
# same than the code store, we have a collision: the code in the
# file has not changed, but the name we have is pointing to a new
# code block.
- if (not old_first_line == first_line
+ if (not old_first_line == first_line
and source_file is not None
and os.path.exists(source_file)):
_, func_name = get_func_name(self.func, resolv_alias=False)
num_lines = len(func_code.split('\n'))
on_disk_func_code = file(source_file).readlines()[
- old_first_line-1:old_first_line-1+num_lines-1]
+ old_first_line - 1:old_first_line - 1 + num_lines - 1]
on_disk_func_code = ''.join(on_disk_func_code)
if on_disk_func_code.rstrip() == old_func_code.rstrip():
warnings.warn(JobLibCollisionWarning(
'Possible name collisions between functions '
"'%s' (%s:%i) and '%s' (%s:%i)" %
- (func_name, source_file, old_first_line,
+ (func_name, source_file, old_first_line,
func_name, source_file, first_line)),
stacklevel=stacklevel)
@@ -303,9 +296,8 @@ def _check_previous_func_code(self, stacklevel=2):
self.clear(warn=True)
return False
-
def clear(self, warn=True):
- """ Empty the function's cache.
+ """ Empty the function's cache.
"""
func_dir = self._get_func_dir(mkdir=False)
if self._verbose and warn:
@@ -321,9 +313,8 @@ def clear(self, warn=True):
func_code_file = os.path.join(func_dir, 'func_code.py')
self._write_func_code(func_code_file, func_code, first_line)
-
def call(self, *args, **kwargs):
- """ Force the execution of the function with the given arguments and
+ """ Force the execution of the function with the given arguments and
persist the output values.
"""
start_time = time.time()
@@ -337,17 +328,16 @@ def call(self, *args, **kwargs):
if self._verbose:
_, name = get_func_name(self.func)
msg = '%s - %s' % (name, format_time(duration))
- print max(0, (80 - len(msg)))*'_' + msg
+ print max(0, (80 - len(msg))) * '_' + msg
return output
-
def format_call(self, *args, **kwds):
- """ Returns a nicely formatted statement displaying the function
+ """ Returns a nicely formatted statement displaying the function
call with the given arguments.
"""
path, signature = self.format_signature(self.func, *args,
**kwds)
- msg = '%s\n[Memory] Calling %s...\n%s' % (80*'_', path, signature)
+ msg = '%s\n[Memory] Calling %s...\n%s' % (80 * '_', path, signature)
return msg
# XXX: Not using logging framework
#self.debug(msg)
@@ -390,7 +380,7 @@ def _persist_output(self, output, dir):
filename = os.path.join(dir, 'output.pkl')
if 'numpy' in sys.modules and self.save_npy:
- numpy_pickle.dump(output, filename)
+ numpy_pickle.dump(output, filename)
else:
output_file = file(filename, 'w')
pickle.dump(output, output_file, protocol=2)
@@ -398,7 +388,6 @@ def _persist_output(self, output, dir):
except OSError:
" Race condition in the creation of the directory "
-
def _persist_input(self, output_dir, *args, **kwargs):
""" Save a small summary of the call using json format in the
output directory.
@@ -433,7 +422,7 @@ def load_output(self, output_dir):
)
filename = os.path.join(output_dir, 'output.pkl')
if self.save_npy:
- return numpy_pickle.load(filename,
+ return numpy_pickle.load(filename,
mmap_mode=self.mmap_mode)
else:
output_file = file(filename, 'r')
@@ -444,7 +433,7 @@ def load_output(self, output_dir):
#-------------------------------------------------------------------------
# Private `object` interface
#-------------------------------------------------------------------------
-
+
def __repr__(self):
return '%s(func=%s, cachedir=%s)' % (
self.__class__.__name__,
@@ -453,14 +442,13 @@ def __repr__(self):
)
-
-################################################################################
+###############################################################################
# class `Memory`
-################################################################################
+###############################################################################
class Memory(Logger):
""" A context object for caching a function's return value each time it
is called with the same input arguments.
-
+
All values are cached on the filesystem, in a deep directory
structure.
@@ -469,7 +457,7 @@ class Memory(Logger):
#-------------------------------------------------------------------------
# Public interface
#-------------------------------------------------------------------------
-
+
def __init__(self, cachedir, save_npy=True, mmap_mode=None,
verbose=1):
"""
@@ -488,7 +476,7 @@ def __init__(self, cachedir, save_npy=True, mmap_mode=None,
arguments. Only used if save_npy was true when the
cache was created.
verbose: int, optional
- Verbosity flag, controls the debug messages that are issued
+ Verbosity flag, controls the debug messages that are issued
as functions are revaluated.
"""
# XXX: Bad explaination of the None value of cachedir
@@ -504,7 +492,6 @@ def __init__(self, cachedir, save_npy=True, mmap_mode=None,
if not os.path.exists(self.cachedir):
os.makedirs(self.cachedir)
-
def cache(self, func=None, ignore=None, verbose=None,
mmap_mode=False):
""" Decorates the given function func to only compute its return
@@ -527,13 +514,13 @@ def cache(self, func=None, ignore=None, verbose=None,
Returns
-------
decorated_func: MemorizedFunc object
- The returned object is a MemorizedFunc object, that is
+ The returned object is a MemorizedFunc object, that is
callable (behaves like a function), but offers extra
methods for cache lookup and management. See the
documentation for :class:`joblib.memory.MemorizedFunc`.
"""
if func is None:
- # Partial application, to be able to specify extra keyword
+ # Partial application, to be able to specify extra keyword
# arguments in decorators
return functools.partial(self.cache, ignore=ignore)
if self.cachedir is None:
@@ -551,7 +538,6 @@ def cache(self, func=None, ignore=None, verbose=None,
verbose=verbose,
timestamp=self.timestamp)
-
def clear(self, warn=True):
""" Erase the complete cache directory.
"""
@@ -559,7 +545,6 @@ def clear(self, warn=True):
self.warn('Flushing completely the cache')
rm_subdirs(self.cachedir)
-
def eval(self, func, *args, **kwargs):
""" Eval function func with arguments `*args` and `**kwargs`,
in the context of the memory.
@@ -576,21 +561,18 @@ def eval(self, func, *args, **kwargs):
#-------------------------------------------------------------------------
# Private `object` interface
#-------------------------------------------------------------------------
-
+
def __repr__(self):
return '%s(cachedir=%s)' % (
self.__class__.__name__,
repr(self.cachedir),
)
-
def __reduce__(self):
""" We don't store the timestamp when pickling, to avoid the hash
depending from it.
In addition, when unpickling, we run the __init__
"""
# We need to remove 'joblib' from the end of cachedir
- return (self.__class__, (self.cachedir[:-7],
+ return (self.__class__, (self.cachedir[:-7],
self.save_npy, self.mmap_mode, self._verbose))
-
-
View
16 joblib/my_exceptions.py
@@ -7,6 +7,7 @@
import sys
+
class JoblibException(Exception):
""" A simple exception with an error message that you can get to.
"""
@@ -21,9 +22,9 @@ def __reduce__(self):
def __repr__(self):
return '%s\n%s\n%s\n%s' % (
self.__class__.__name__,
- 75*'_',
+ 75 * '_',
self.message,
- 75*'_')
+ 75 * '_')
__str__ = __repr__
@@ -32,19 +33,19 @@ class TransportableException(JoblibException):
""" An exception containing all the info to wrap an original
exception and recreate it.
"""
-
+
def __init__(self, message, etype):
self.message = message
- self.etype = etype
+ self.etype = etype
def __reduce__(self):
# For pickling
return self.__class__, (self.message, self.etype), {}
-
_exception_mapping = dict()
+
def _mk_exception(exception, name=None):
# Create an exception inheriting from both JoblibException
# and that exception
@@ -55,7 +56,7 @@ def _mk_exception(exception, name=None):
# Avoid creating twice the same exception
this_exception = _exception_mapping[this_name]
else:
- this_exception = type(this_name, (exception, JoblibException),
+ this_exception = type(this_name, (exception, JoblibException),
dict(__repr__=JoblibException.__repr__,
__str__=JoblibException.__str__),
)
@@ -88,7 +89,6 @@ def _mk_common_exceptions():
return namespace
-# Updating module locals so that the exceptions pickle right. AFAIK this
+# Updating module locals so that the exceptions pickle right. AFAIK this
# works only at module-creation time
locals().update(_mk_common_exceptions())
-
View
30 joblib/numpy_pickle.py
@@ -2,22 +2,24 @@
A pickler to save numpy arrays in separate .npy files.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.
import pickle
import traceback
-import sys, os
+import sys
+import os
if sys.version_info[0] == 3:
from pickle import _Unpickler as Unpickler
else:
from pickle import Unpickler
-################################################################################
+###############################################################################
# Utility objects for persistence.
+
class NDArrayWrapper(object):
""" An object to be persisted instead of numpy arrays.
@@ -28,7 +30,7 @@ def __init__(self, filename):
self.filename = filename
-################################################################################
+###############################################################################
# Pickler classes
class NumpyPickler(pickle.Pickler):
@@ -50,14 +52,14 @@ def __init__(self, filename):
def save(self, obj):
""" Subclass the save method, to save ndarray subclasses in npy
- files, rather than pickling them. Off course, this is a
+ files, rather than pickling them. Off course, this is a
total abuse of the Pickler class.
"""
if isinstance(obj, self.np.ndarray):
self._npy_counter += 1
try:
filename = '%s_%02i.npy' % (self._filename,
- self._npy_counter )
+ self._npy_counter)
self._filenames.append(filename)
self.np.save(filename, obj)
obj = NDArrayWrapper(os.path.basename(filename))
@@ -70,7 +72,6 @@ def save(self, obj):
pickle.Pickler.save(self, obj)
-
class NumpyUnpickler(Unpickler):
""" A subclass of the Unpickler to unpickle our numpy pickles.
"""
@@ -79,17 +80,16 @@ class NumpyUnpickler(Unpickler):
def __init__(self, filename, mmap_mode=None):
self._filename = filename
self.mmap_mode = mmap_mode
- self._dirname = os.path.dirname(filename)
+ self._dirname = os.path.dirname(filename)
self.file = open(filename, 'rb')
Unpickler.__init__(self, self.file)
import numpy as np
self.np = np
-
def load_build(self):
""" This method is called to set the state of a knewly created
- object.
-
+ object.
+
We capture it to replace our place-holder objects,
NDArrayWrapper, by the array we are interested in. We
replace directly in the stack of pickler.
@@ -107,16 +107,15 @@ def load_build(self):
nd_array_wrapper.filename))
self.stack.append(array)
-
# Be careful to register our new method.
dispatch[pickle.BUILD] = load_build
-################################################################################
+###############################################################################
# Utility functions
def dump(value, filename):
- """ Persist an arbitrary Python object into a filename, with numpy arrays
+ """ Persist an arbitrary Python object into a filename, with numpy arrays
saved as separate .npy files.
See Also
@@ -134,7 +133,7 @@ def dump(value, filename):
def load(filename, mmap_mode=None):
- """ Reconstruct a Python object and the numpy arrays it contains from
+ """ Reconstruct a Python object and the numpy arrays it contains from
a persisted file.
This function loads the numpy array files saved separately. If
@@ -153,4 +152,3 @@ def load(filename, mmap_mode=None):
if 'unpickler' in locals() and hasattr(unpickler, 'file'):
unpickler.file.close()
return obj
-
View
100 joblib/parallel.py
@@ -30,7 +30,8 @@
from .logger import Logger, short_format_time
from .my_exceptions import TransportableException, _mk_exception
-################################################################################
+
+###############################################################################
# CPU that works also when multiprocessing is not installed (python2.5)
def cpu_count():
""" Return the number of CPUs.
@@ -40,9 +41,7 @@ def cpu_count():
return multiprocessing.cpu_count()
-
-
-################################################################################
+###############################################################################
class WorkerInterrupt(Exception):
""" An exception that is not KeyboardInterrupt to allow subprocesses
to be interrupted.
@@ -50,18 +49,16 @@ class WorkerInterrupt(Exception):
pass
-################################################################################
+###############################################################################
class SafeFunction(object):
""" Wraps a function to make it exception with full traceback in
their representation.
- Useful for parallel computing with multiprocessing, for which
+ Useful for parallel computing with multiprocessing, for which
exceptions cannot be captured.
"""
-
def __init__(self, func):
self.func = func
-
def __call__(self, *args, **kwargs):
try:
return self.func(*args, **kwargs)
@@ -77,7 +74,7 @@ def __call__(self, *args, **kwargs):
raise TransportableException(text, e_type)
-################################################################################
+###############################################################################
def delayed(function):
""" Decorator used to capture the arguments of a function.
"""
@@ -94,22 +91,22 @@ def delayed_function(*args, **kwargs):
return delayed_function
-################################################################################
+###############################################################################
class ImmediateApply(object):
""" A non-delayed apply function.
"""
- def __init__ (self, func, args, kwargs):
+ def __init__(self, func, args, kwargs):
# Don't delay the application, to avoid keeping the input
# arguments in memory
self.results = func(*args, **kwargs)
- def get (self):
+ def get(self):
return self.results
-################################################################################
+###############################################################################
class CallBack(object):
- """ Callback used by parallel: it is used for progress reporting, and
+ """ Callback used by parallel: it is used for progress reporting, and
to add data to be processed
"""
def __init__(self, index, parallel):
@@ -126,15 +123,15 @@ def print_progress(self):
# XXX: Not using the logger framework: need to
# learn to use logger better.
n_jobs = len(self.parallel._pool._pool)
- if self.parallel.n_dispatched > 2*n_jobs:
+ if self.parallel.n_dispatched > 2 * n_jobs:
# Report less often
if not self.index % n_jobs == 0:
return
elapsed_time = time.time() - self.parallel._start_time
- remaining_time = (elapsed_time/(self.index + 1)*
+ remaining_time = (elapsed_time / (self.index + 1) *
(self.parallel.n_dispatched - self.index - 1.))
if self.parallel._iterable:
- # The object is still building it's job list
+ # The object is still building its job list
total = "%3i+" % self.parallel.n_dispatched
else:
total = "%3i " % self.parallel.n_dispatched
@@ -145,14 +142,14 @@ def print_progress(self):
writer = sys.stdout.write
writer('[%s]: Done %3i out of %s |elapsed: %s remaining: %s\n'
% (self.parallel,
- self.index+1,
- total,
+ self.index + 1,
+ total,
short_format_time(elapsed_time),
short_format_time(remaining_time),
))
-################################################################################
+###############################################################################
class Parallel(Logger):
''' Helper class for readable parallel mapping.
@@ -214,7 +211,7 @@ class Parallel(Logger):
(0.0, 0.5, 0.0, 0.5, 0.0, 0.5, 0.0, 0.5, 0.0, 0.5)
>>> i
(0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0)
-
+
The progress meter::
>>> from time import sleep
@@ -236,7 +233,7 @@ class Parallel(Logger):
>>> Parallel(n_jobs=2)(delayed(atoi)(n) for n in ('1', '300', 30)) #doctest: +SKIP
#...
---------------------------------------------------------------------------
- Sub-process traceback:
+ Sub-process traceback:
---------------------------------------------------------------------------
TypeError Fri Jul 2 20:32:05 2010
PID: 4151 Python 2.6.5: /usr/bin/python
@@ -245,14 +242,14 @@ class Parallel(Logger):
398 is chosen from the leading characters of s, 0 for octal, 0x or
399 0X for hexadecimal. If base is 16, a preceding 0x or 0X is
400 accepted.
- 401
+ 401
402 """
--> 403 return _int(s, base)
- 404
- 405
+ 404
+ 405
406 # Convert string to long integer
407 def atol(s, base=10):
-
+
TypeError: int() can't convert non-string with explicit base
___________________________________________________________________________
@@ -269,7 +266,7 @@ class Parallel(Logger):
... for i in range(6):
... print 'Produced %s' % i
... yield i
-
+
>>> out = Parallel(n_jobs=2, verbose=100, pre_dispatch='1.5*n_jobs')(
... delayed(sqrt)(i) for i in producer()) #doctest: +SKIP
Produced 0
@@ -283,24 +280,22 @@ class Parallel(Logger):
'''
def __init__(self, n_jobs=None, verbose=0, pre_dispatch='all'):
self.verbose = verbose
- self.n_jobs = n_jobs
- self.pre_dispatch = pre_dispatch
- self._pool = None
- # Not starting the pool in the __init__ is a design decision, to
- # be able to close it ASAP, and not burden the user with closing
- # it.
+ self.n_jobs = n_jobs
+ self.pre_dispatch = pre_dispatch
+ self._pool = None
+ # Not starting the pool in the __init__ is a design decision, to be
+ # able to close it ASAP, and not burden the user with closing it.
self._output = None
- self._jobs = list()
-
+ self._jobs = list()
def dispatch(self, func, args, kwargs):
- """ Queue the function for computing, with or without multiprocessing
+ """ Queue the function for computing, with or without multiprocessing
"""
if self._pool is None:
job = ImmediateApply(func, args, kwargs)
if self.verbose:
print '[%s]: Done job %3i | elapsed: %s' % (
- self, len(self._jobs)+1,
+ self, len(self._jobs) + 1,
short_format_time(time.time() - self._start_time)
)
self._jobs.append(job)
@@ -318,14 +313,13 @@ def dispatch(self, func, args, kwargs):
finally:
self._lock.release()
-
def dispatch_next(self):
""" Dispatch more data for parallel processing
"""
self._dispatch_amount += 1
while self._dispatch_amount:
try:
- # XXX: possible race condition shuffling the order of
+ # XXX: possible race condition shuffling the order of
# dispatchs in the next two lines.
func, args, kwargs = self._iterable.next()
self.dispatch(func, args, kwargs)
@@ -338,7 +332,6 @@ def dispatch_next(self):
self._iterable = None
return
-
def retrieve(self):
self._output = list()
while self._jobs:
@@ -352,7 +345,7 @@ def retrieve(self):
try:
self._output.append(job.get())
except tuple(self.exceptions), exception:
- if isinstance(exception,
+ if isinstance(exception,
(KeyboardInterrupt, WorkerInterrupt)):
# We have captured a user interruption, clean up
# everything
@@ -361,17 +354,14 @@ def retrieve(self):
self._pool.terminate()
raise exception
elif isinstance(exception, TransportableException):
- # Capture exception to add information on
- # the local stack in addition to the distant
- # stack
- this_report = format_outer_frames(
- context=10,
- stack_start=1,
- )
+ # Capture exception to add information on the local stack
+ # in addition to the distant stack
+ this_report = format_outer_frames(context=10,
+ stack_start=1)
report = """Multiprocessing exception:
%s
---------------------------------------------------------------------------
-Sub-process traceback:
+Sub-process traceback:
---------------------------------------------------------------------------
%s""" % (
this_report,
@@ -382,7 +372,6 @@ def retrieve(self):
raise exception_type(report)
raise exception
-
def __call__(self, iterable):
if self._jobs:
raise ValueError('This Parallel instance is already running')
@@ -418,7 +407,7 @@ def __call__(self, iterable):
try:
for function, args, kwargs in iterable:
self.dispatch(function, args, kwargs)
-
+
self.retrieve()
finally:
if n_jobs > 1:
@@ -429,12 +418,5 @@ def __call__(self, iterable):
self._output = None
return output
-
def __repr__(self):
- return '%s(n_jobs=%s)' % (
- self.__class__.__name__,
- self.n_jobs,
- )
-
-
-
+ return '%s(n_jobs=%s)' % ( self.__class__.__name__, self.n_jobs)
View
1  joblib/test/__init__.py
@@ -1 +0,0 @@
-
View
5 joblib/test/common.py
@@ -6,12 +6,13 @@
# A decorator to run tests only when numpy is available
try:
import numpy as np
+
def with_numpy(func):
""" A decorator to skip tests requiring numpy.
"""
return func
-except ImportError:
+except ImportError:
def with_numpy(func):
""" A decorator to skip tests requiring numpy.
"""
@@ -19,5 +20,3 @@ def my_func():
raise nose.SkipTest('Test requires numpy')
return my_func
np = None
-
-
View
12 joblib/test/test_disk.py
@@ -2,7 +2,7 @@
Unit tests for the disk utilities.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2010 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -15,7 +15,8 @@
from ..disk import memstr_to_kbytes, disk_used
-################################################################################
+
+###############################################################################
def test_disk_used():
cachedir = mkdtemp()
@@ -29,8 +30,8 @@ def test_disk_used():
a = array.array('i')
sizeof_i = a.itemsize
target_size = 1024
- n = target_size*1024/sizeof_i
- a = array.array('i', n*(1,))
+ n = target_size * 1024 / sizeof_i
+ a = array.array('i', n * (1,))
a.tofile(file(os.path.join(cachedir, 'test'), 'w'))
nose.tools.assert_true(disk_used(cachedir) >= target_size)
nose.tools.assert_true(disk_used(cachedir) < target_size + 12)
@@ -40,8 +41,7 @@ def test_disk_used():
def test_memstr_to_kbytes():
for text, value in zip(('80G', '1.4M', '120M', '53K'),
- (80*1024**2, int(1.4*1024), 120*1024, 53)):
+ (80 * 1024**2, int(1.4 * 1024), 120 * 1024, 53)):
yield nose.tools.assert_equal, memstr_to_kbytes(text), value
nose.tools.assert_raises(ValueError, memstr_to_kbytes, 'foobar')
-
View
6 joblib/test/test_format_stack.py
@@ -1,8 +1,8 @@
"""
-Unit tests for the stack formatting utilities
+Unit tests for the stack formatting utilities
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2010 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -11,7 +11,7 @@
from ..format_stack import safe_repr
-################################################################################
+###############################################################################
class Vicious(object):
def __repr__(self):
View
65 joblib/test/test_func_inspect.py
@@ -2,7 +2,7 @@
Test the func_inspect module.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -13,41 +13,51 @@
from ..func_inspect import filter_args, get_func_name, get_func_code
from ..memory import Memory
-################################################################################
-# Module-level functions, for tests
+
+###############################################################################
+# Module-level functions, for tests
def f(x, y=0):
pass
+
def f2(x):
pass
+
# Create a Memory object to test decorated functions.
# We should be careful not to call the decorated functions, so that
# cache directories are not created in the temp dir.
mem = Memory(cachedir=tempfile.gettempdir())
+
@mem.cache
def g(x):
return x
+
def h(x, y=0, *args, **kwargs):
pass
+
def i(x=1):
pass
+
def j(x, y, **kwargs):
pass
+
def k(*args, **kwargs):
pass
+
class Klass(object):
def f(self, x):
return x
-################################################################################
+
+###############################################################################
# Tests
def test_filter_args():
@@ -56,8 +66,8 @@ def test_filter_args():
yield nose.tools.assert_equal, filter_args(f, ['y'], 0), {'x': 0}
yield nose.tools.assert_equal, filter_args(f, ['y'], 0, y=1), {'x': 0}
yield nose.tools.assert_equal, filter_args(f, ['x', 'y'], 0), {}
- yield nose.tools.assert_equal, filter_args(f, [], 0, y=1), {'x':0, 'y':1}
- yield nose.tools.assert_equal, filter_args(f, ['y'], x=2, y=1), {'x':2}
+ yield nose.tools.assert_equal, filter_args(f, [], 0, y=1), {'x': 0, 'y': 1}
+ yield nose.tools.assert_equal, filter_args(f, ['y'], x=2, y=1), {'x': 2}
yield nose.tools.assert_equal, filter_args(i, [], 2), {'x': 2}
yield nose.tools.assert_equal, filter_args(f2, [], x=1), {'x': 1}
@@ -65,41 +75,41 @@ def test_filter_args():
def test_filter_args_method():
obj = Klass()
- nose.tools.assert_equal(filter_args(obj.f, [], 1),
+ nose.tools.assert_equal(filter_args(obj.f, [], 1),
{'x': 1, 'self': obj})
def test_filter_varargs():
yield nose.tools.assert_equal, filter_args(h, [], 1), \
- {'x': 1, 'y': 0, '*':[], '**':{}}
+ {'x': 1, 'y': 0, '*': [], '**': {}}
yield nose.tools.assert_equal, filter_args(h, [], 1, 2, 3, 4), \
- {'x': 1, 'y': 2, '*':[3, 4], '**':{}}
+ {'x': 1, 'y': 2, '*': [3, 4], '**': {}}
yield nose.tools.assert_equal, filter_args(h, [], 1, 25, ee=2), \
- {'x': 1, 'y': 25, '*':[], '**':{'ee':2}}
+ {'x': 1, 'y': 25, '*': [], '**': {'ee': 2}}
yield nose.tools.assert_equal, filter_args(h, ['*'], 1, 2, 25, ee=2), \
- {'x': 1, 'y': 2, '**':{'ee':2}}
+ {'x': 1, 'y': 2, '**': {'ee': 2}}
def test_filter_kwargs():
- nose.tools.assert_equal(filter_args(k, [], 1, 2, ee=2),
- {'*': [1, 2], '**':{'ee':2}})
- nose.tools.assert_equal(filter_args(k, [], 3, 4),
- {'*': [3, 4], '**':{}})
+ nose.tools.assert_equal(filter_args(k, [], 1, 2, ee=2),
+ {'*': [1, 2], '**': {'ee': 2}})
+ nose.tools.assert_equal(filter_args(k, [], 3, 4),
+ {'*': [3, 4], '**': {}})
def test_filter_args_2():
- nose.tools.assert_equal(filter_args(j, [], 1, 2, ee=2),
- {'x': 1, 'y': 2, '**':{'ee':2}})
-
+ nose.tools.assert_equal(filter_args(j, [], 1, 2, ee=2),
+ {'x': 1, 'y': 2, '**': {'ee': 2}})
+
nose.tools.assert_raises(ValueError, filter_args, f, 'a', None)
# Check that we capture an undefined argument
nose.tools.assert_raises(ValueError, filter_args, f, ['a'], None)
ff = functools.partial(f, 1)
# filter_args has to special-case partial
- nose.tools.assert_equal(filter_args(ff, [], 1),
- {'*': [1], '**':{}})
- nose.tools.assert_equal(filter_args(ff, ['y'], 1),
- {'*': [1], '**':{}})
+ nose.tools.assert_equal(filter_args(ff, [], 1),
+ {'*': [1], '**': {}})
+ nose.tools.assert_equal(filter_args(ff, ['y'], 1),
+ {'*': [1], '**': {}})
def test_func_name():
@@ -114,26 +124,25 @@ def test_func_inspect_errors():
nose.tools.assert_equal(get_func_name('a'.lower)[-1], 'lower')
nose.tools.assert_equal(get_func_code('a'.lower)[1:], (None, -1))
ff = lambda x: x
- nose.tools.assert_equal(get_func_name(ff, win_characters=False)[-1],
+ nose.tools.assert_equal(get_func_name(ff, win_characters=False)[-1],
'<lambda>')
- nose.tools.assert_equal(get_func_code(ff)[1],
+ nose.tools.assert_equal(get_func_code(ff)[1],
__file__.replace('.pyc', '.py'))
# Simulate a function defined in __main__
ff.__module__ = '__main__'
- nose.tools.assert_equal(get_func_name(ff, win_characters=False)[-1],
+ nose.tools.assert_equal(get_func_name(ff, win_characters=False)[-1],
'<lambda>')
- nose.tools.assert_equal(get_func_code(ff)[1],
+ nose.tools.assert_equal(get_func_code(ff)[1],
__file__.replace('.pyc', '.py'))
-
def test_bound_methods():
""" Make sure that calling the same method on two different instances
of the same class does resolv to different signatures.
"""
a = Klass()
b = Klass()
- nose.tools.assert_not_equal(filter_args(a.f, [], 1),
+ nose.tools.assert_not_equal(filter_args(a.f, [], 1),
filter_args(b.f, [], 1))
View
18 joblib/test/test_logger.py
@@ -2,7 +2,7 @@
Test the logger module.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -17,10 +17,12 @@
from ..logger import PrintTime
-################################################################################
+
+###############################################################################
# Test fixtures
env = dict()
+
def setup():
""" Test setup.
"""
@@ -28,7 +30,7 @@ def setup():
if os.path.exists(cachedir):
shutil.rmtree(cachedir)
env['dir'] = cachedir
-
+
def teardown():
""" Test teardown.
@@ -37,7 +39,7 @@ def teardown():
shutil.rmtree(env['dir'])
-################################################################################
+###############################################################################
# Tests
def test_print_time():
""" A simple smoke test for PrintTime.
@@ -50,15 +52,15 @@ def test_print_time():
# Create a second time, to smoke test log rotation.
print_time = PrintTime(logfile=os.path.join(env['dir'], 'test.log'))
print_time('Foo')
- # And a third time
+ # And a third time
print_time = PrintTime(logfile=os.path.join(env['dir'], 'test.log'))
print_time('Foo')
printed_text = sys.stderr.getvalue()
# Use regexps to be robust to time variations
- match = r"Foo: 0\..s, 0\.0min\nFoo: 0\..s, 0.0min\nFoo: .\..s, 0.0min\n"
+ match = r"Foo: 0\..s, 0\.0min\nFoo: 0\..s, 0.0min\nFoo: " + \
+ r".\..s, 0.0min\n"
if not re.match(match, printed_text):
- raise AssertionError('Excepted %s, got %s' %
+ raise AssertionError('Excepted %s, got %s' %
(match, printed_text))
finally:
sys.stderr = orig_stderr
-
View
57 joblib/test/test_memory.py
@@ -2,7 +2,7 @@
Test the memory module.
"""
-# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
+# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.
@@ -19,7 +19,8 @@
from ..memory import Memory, MemorizedFunc
from .common import with_numpy, np
-################################################################################
+
+###############################################################################