18 sklearn/externals/joblib/func_inspect.py
View file
@@ -11,6 +11,7 @@
import warnings
import os


def get_func_code(func):
""" Attempts to retrieve a reliable function code hash.
@@ -39,7 +40,8 @@ def get_func_code(func):
source_file_obj = file(source_file)
first_line = func.func_code.co_firstlineno
# All the lines after the function definition:
source_lines = list(itertools.islice(source_file_obj, first_line-1, None))
source_lines = list(itertools.islice(source_file_obj, first_line - 1,
None))
return ''.join(inspect.getblock(source_lines)), source_file, first_line
except:
# If the source code fails, we use the hash. This is fragile and
@@ -87,7 +89,9 @@ def get_func_name(func, resolv_alias=True, win_characters=True):
except:
filename = None
if filename is not None:
filename = filename.replace('/', '-')
# mangling of full path to filename
filename = filename.replace(os.sep, '-')
filename = filename.replace(":", "-")
if filename.endswith('.py'):
filename = filename[:-3]
module = module + '-' + filename
@@ -150,14 +154,14 @@ def filter_args(func, ignore_lst, *args, **kwargs):
if ignore_lst:
warnings.warn('Cannot inspect object %s, ignore list will '
'not work.' % func, stacklevel=2)
return {'*':args, '**':kwargs}
return {'*': args, '**': kwargs}
arg_spec = inspect.getargspec(func)
# We need to if/them to account for different versions of Python
if hasattr(arg_spec, 'args'):
arg_names = arg_spec.args
arg_names = arg_spec.args
arg_defaults = arg_spec.defaults
arg_keywords = arg_spec.keywords
arg_varargs = arg_spec.varargs
arg_varargs = arg_spec.varargs
else:
arg_names, arg_varargs, arg_keywords, arg_defaults = arg_spec
arg_defaults = arg_defaults or {}
@@ -195,7 +199,6 @@ def filter_args(func, ignore_lst, *args, **kwargs):
)
)


varkwargs = dict()
for arg_name, arg_value in kwargs.iteritems():
if arg_name in arg_dict:
@@ -209,7 +212,7 @@ def filter_args(func, ignore_lst, *args, **kwargs):
if arg_keywords is not None:
arg_dict['**'] = varkwargs
if arg_varargs is not None:
varargs = args[arg_position+1:]
varargs = args[arg_position + 1:]
arg_dict['*'] = varargs

# Now remove the arguments to be ignored
@@ -227,4 +230,3 @@ def filter_args(func, ignore_lst, *args, **kwargs):
)))
# XXX: Return a sorted list of pairs?
return arg_dict

3 sklearn/externals/joblib/hashing.py
View file
@@ -13,6 +13,7 @@
import cStringIO
import types


class Hasher(pickle.Pickler):
""" A subclass of pickler, to do cryptographic hashing, rather than
pickling.
@@ -41,6 +42,7 @@ def save(self, obj):
obj = (func_name, inst, cls)
pickle.Pickler.save(self, obj)


class NumpyHasher(Hasher):
""" Special case the hasher for when numpy is loaded.
"""
@@ -112,4 +114,3 @@ def hash(obj, hash_name='md5', coerce_mmap=False):
else:
hasher = Hasher(hash_name=hash_name)
return hasher.hash(obj)

53 sklearn/externals/joblib/logger.py
View file
@@ -16,6 +16,9 @@
import logging
import pprint

from .disk import mkdirp


def _squeeze_time(t):
"""Remove .1s to the time under Windows: this is the time it take to
stat files. This is needed to make results similar to timings under
@@ -26,20 +29,23 @@ def _squeeze_time(t):
else:
return t


def format_time(t):
t = _squeeze_time(t)
return "%.1fs, %.1fmin" % (t, t/60.)
return "%.1fs, %.1fmin" % (t, t / 60.)


def short_format_time(t):
t = _squeeze_time(t)
if t > 60:
return "%4.1fmin" % (t/60.)
return "%4.1fmin" % (t / 60.)
else:
return " %5.1fs" % (t)

################################################################################

###############################################################################
# class `Logger`
################################################################################
###############################################################################
class Logger(object):
""" Base class for logging messages.
"""
@@ -75,9 +81,9 @@ def format(self, obj, indent=0):
return out


################################################################################
###############################################################################
# class `PrintTime`
################################################################################
###############################################################################
class PrintTime(object):
""" Print and log messages while keeping track of time.
"""
@@ -92,47 +98,45 @@ def __init__(self, logfile=None, logdir=None):
logfile = os.path.join(logdir, 'joblib.log')
self.logfile = logfile
if logfile is not None:
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
mkdirp(os.path.dirname(logfile))
if os.path.exists(logfile):
# Rotate the logs
for i in range(1, 9):
if os.path.exists(logfile+'.%i' % i):
try:
shutil.move(logfile+'.%i' % i,
logfile+'.%i' % (i+1))
except:
"No reason failing here"
for i in xrange(1, 9):
try:
shutil.move(logfile + '.%i' % i,
logfile + '.%i' % (i + 1))
except:
"No reason failing here"
# Use a copy rather than a move, so that a process
# monitoring this file does not get lost.
try:
shutil.copy(logfile, logfile+'.1')
shutil.copy(logfile, logfile + '.1')
except:
"No reason failing here"
try:
logfile = file(logfile, 'w')
logfile.write('\nLogging joblib python script\n')
logfile.write('\n---%s---\n' % time.ctime(self.last_time))
with open(logfile, 'w') as logfile:
logfile.write('\nLogging joblib python script\n')
logfile.write('\n---%s---\n' % time.ctime(self.last_time))
except:
""" Multiprocessing writing to files can create race
conditions. Rather fail silently than crash the
caculation.
computation.
"""
# XXX: We actually need a debug flag to disable this
# silent failure.


def __call__(self, msg='', total=False):
""" Print the time elapsed between the last call and the current
call, with an optional message.
"""
if not total:
time_lapse = time.time() - self.last_time
full_msg = "%s: %s" % (msg, format_time(time_lapse) )
full_msg = "%s: %s" % (msg, format_time(time_lapse))
else:
# FIXME: Too much logic duplicated
time_lapse = time.time() - self.start_time
full_msg = "%s: %.2fs, %.1f min" % (msg, time_lapse, time_lapse/60)
full_msg = "%s: %.2fs, %.1f min" % (msg, time_lapse,
time_lapse / 60)
print >> sys.stderr, full_msg
if self.logfile is not None:
try:
@@ -145,6 +149,3 @@ def __call__(self, msg='', total=False):
# XXX: We actually need a debug flag to disable this
# silent failure.
self.last_time = time.time()



88 sklearn/externals/joblib/memory.py
View file
@@ -9,6 +9,7 @@
# License: BSD Style, 3 clauses.


from __future__ import with_statement
import os
import shutil
import sys
@@ -37,7 +38,7 @@
from .func_inspect import get_func_code, get_func_name, filter_args
from .logger import Logger, format_time
from . import numpy_pickle
from .disk import rm_subdirs
from .disk import mkdirp, rm_subdirs

FIRST_LINE_TEXT = "# first line:"

@@ -72,9 +73,9 @@ class JobLibCollisionWarning(UserWarning):
"""


################################################################################
###############################################################################
# class `MemorizedFunc`
################################################################################
###############################################################################
class MemorizedFunc(Logger):
""" Callable object decorating a function for caching its return value
each time it is called.
@@ -142,8 +143,7 @@ def __init__(self, func, cachedir, ignore=None, save_npy=True,
if ignore is None:
ignore = []
self.ignore = ignore
if not os.path.exists(self.cachedir):
os.makedirs(self.cachedir)
mkdirp(self.cachedir)
try:
functools.update_wrapper(self, func)
except:
@@ -156,7 +156,6 @@ def __init__(self, func, cachedir, ignore=None, save_npy=True,
doc = func.__doc__
self.__doc__ = 'Memoized version of %s' % doc


def __call__(self, *args, **kwargs):
# Compare the function code with the previous to see if the
# function code has changed
@@ -173,20 +172,17 @@ def __call__(self, *args, **kwargs):
t = time.time() - t0
_, name = get_func_name(self.func)
msg = '%s cache loaded - %s' % (name, format_time(t))
print max(0, (80 - len(msg)))*'_' + msg
print max(0, (80 - len(msg))) * '_' + msg
return out
except Exception:
# XXX: Should use an exception logger
self.warn(
'Exception while loading results for '
'(args=%s, kwargs=%s)\n %s' %
(args, kwargs, traceback.format_exc())
)
self.warn('Exception while loading results for '
'(args=%s, kwargs=%s)\n %s' %
(args, kwargs, traceback.format_exc()))

shutil.rmtree(output_dir, ignore_errors=True)
return self.call(*args, **kwargs)


def __reduce__(self):
""" We don't store the timestamp when pickling, to avoid the hash
depending from it.
@@ -206,17 +202,10 @@ def _get_func_dir(self, mkdir=True):
module, name = get_func_name(self.func)
module.append(name)
func_dir = os.path.join(self.cachedir, *module)
if mkdir and not os.path.exists(func_dir):
try:
os.makedirs(func_dir)
except OSError:
""" Dir exists: we have a race condition here, when using
multiprocessing.
"""
# XXX: Ugly
if mkdir:
mkdirp(func_dir)
return func_dir


def get_output_dir(self, *args, **kwargs):
""" Returns the directory in which are persisted the results
of the function corresponding to the given arguments.
@@ -228,16 +217,15 @@ def get_output_dir(self, *args, **kwargs):
*args, **kwargs),
coerce_mmap=coerce_mmap)
output_dir = os.path.join(self._get_func_dir(self.func),
argument_hash)
argument_hash)
return output_dir, argument_hash


def _write_func_code(self, filename, func_code, first_line):
""" Write the function code and the filename to a file.
"""
func_code = '%s %i\n%s' % (FIRST_LINE_TEXT, first_line, func_code)
file(filename, 'w').write(func_code)

with open(filename, 'w') as out:
out.write(func_code)

def _check_previous_func_code(self, stacklevel=2):
"""
@@ -252,10 +240,9 @@ def _check_previous_func_code(self, stacklevel=2):
func_code_file = os.path.join(func_dir, 'func_code.py')

try:
if not os.path.exists(func_code_file):
raise IOError
old_func_code, old_first_line = \
extract_first_line(file(func_code_file).read())
with open(func_code_file) as infile:
old_func_code, old_first_line = \
extract_first_line(infile.read())
except IOError:
self._write_func_code(func_code_file, func_code, first_line)
return False
@@ -288,7 +275,7 @@ def _check_previous_func_code(self, stacklevel=2):
_, func_name = get_func_name(self.func, resolv_alias=False)
num_lines = len(func_code.split('\n'))
on_disk_func_code = file(source_file).readlines()[
old_first_line-1:old_first_line-1+num_lines-1]
old_first_line - 1:old_first_line - 1 + num_lines - 1]
on_disk_func_code = ''.join(on_disk_func_code)
if on_disk_func_code.rstrip() == old_func_code.rstrip():
warnings.warn(JobLibCollisionWarning(
@@ -303,7 +290,6 @@ def _check_previous_func_code(self, stacklevel=2):
self.clear(warn=True)
return False


def clear(self, warn=True):
""" Empty the function's cache.
"""
@@ -312,16 +298,11 @@ def clear(self, warn=True):
self.warn("Clearing cache %s" % func_dir)
if os.path.exists(func_dir):
shutil.rmtree(func_dir, ignore_errors=True)
try:
os.makedirs(func_dir)
except OSError:
""" Directory exists: it has been created by another process
in the mean time. """
mkdirp(func_dir)
func_code, _, first_line = get_func_code(self.func)
func_code_file = os.path.join(func_dir, 'func_code.py')
self._write_func_code(func_code_file, func_code, first_line)


def call(self, *args, **kwargs):
""" Force the execution of the function with the given arguments and
persist the output values.
@@ -337,17 +318,16 @@ def call(self, *args, **kwargs):
if self._verbose:
_, name = get_func_name(self.func)
msg = '%s - %s' % (name, format_time(duration))
print max(0, (80 - len(msg)))*'_' + msg
print max(0, (80 - len(msg))) * '_' + msg
return output


def format_call(self, *args, **kwds):
""" Returns a nicely formatted statement displaying the function
call with the given arguments.
"""
path, signature = self.format_signature(self.func, *args,
**kwds)
msg = '%s\n[Memory] Calling %s...\n%s' % (80*'_', path, signature)
msg = '%s\n[Memory] Calling %s...\n%s' % (80 * '_', path, signature)
return msg
# XXX: Not using logging framework
#self.debug(msg)
@@ -385,20 +365,17 @@ def _persist_output(self, output, dir):
""" Persist the given output tuple in the directory.
"""
try:
if not os.path.exists(dir):
os.makedirs(dir)
mkdirp(dir)
filename = os.path.join(dir, 'output.pkl')

if 'numpy' in sys.modules and self.save_npy:
numpy_pickle.dump(output, filename)
else:
output_file = file(filename, 'w')
pickle.dump(output, output_file, protocol=2)
output_file.close()
with open(filename, 'w') as output_file:
pickle.dump(output, output_file, protocol=2)
except OSError:
" Race condition in the creation of the directory "


def _persist_input(self, output_dir, *args, **kwargs):
""" Save a small summary of the call using json format in the
output directory.
@@ -411,8 +388,7 @@ def _persist_input(self, output_dir, *args, **kwargs):
# This can fail do to race-conditions with multiple
# concurrent joblibs removing the file or the directory
try:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
mkdirp(output_dir)
json.dump(
input_repr,
file(os.path.join(output_dir, 'input_args.json'), 'w'),
@@ -453,10 +429,9 @@ def __repr__(self):
)



################################################################################
###############################################################################
# class `Memory`
################################################################################
###############################################################################
class Memory(Logger):
""" A context object for caching a function's return value each time it
is called with the same input arguments.
@@ -501,9 +476,7 @@ def __init__(self, cachedir, save_npy=True, mmap_mode=None,
self.cachedir = None
else:
self.cachedir = os.path.join(cachedir, 'joblib')
if not os.path.exists(self.cachedir):
os.makedirs(self.cachedir)

mkdirp(self.cachedir)

def cache(self, func=None, ignore=None, verbose=None,
mmap_mode=False):
@@ -551,15 +524,13 @@ def cache(self, func=None, ignore=None, verbose=None,
verbose=verbose,
timestamp=self.timestamp)


def clear(self, warn=True):
""" Erase the complete cache directory.
"""
if warn:
self.warn('Flushing completely the cache')
rm_subdirs(self.cachedir)


def eval(self, func, *args, **kwargs):
""" Eval function func with arguments `*args` and `**kwargs`,
in the context of the memory.
@@ -583,7 +554,6 @@ def __repr__(self):
repr(self.cachedir),
)


def __reduce__(self):
""" We don't store the timestamp when pickling, to avoid the hash
depending from it.
@@ -592,5 +562,3 @@ def __reduce__(self):
# We need to remove 'joblib' from the end of cachedir
return (self.__class__, (self.cachedir[:-7],
self.save_npy, self.mmap_mode, self._verbose))


10 sklearn/externals/joblib/my_exceptions.py
View file
@@ -7,6 +7,7 @@

import sys


class JoblibException(Exception):
""" A simple exception with an error message that you can get to.
"""
@@ -21,9 +22,9 @@ def __reduce__(self):
def __repr__(self):
return '%s\n%s\n%s\n%s' % (
self.__class__.__name__,
75*'_',
75 * '_',
self.message,
75*'_')
75 * '_')

__str__ = __repr__

@@ -35,16 +36,16 @@ class TransportableException(JoblibException):

def __init__(self, message, etype):
self.message = message
self.etype = etype
self.etype = etype

def __reduce__(self):
# For pickling
return self.__class__, (self.message, self.etype), {}



_exception_mapping = dict()


def _mk_exception(exception, name=None):
# Create an exception inheriting from both JoblibException
# and that exception
@@ -91,4 +92,3 @@ def _mk_common_exceptions():
# Updating module locals so that the exceptions pickle right. AFAIK this
# works only at module-creation time
locals().update(_mk_common_exceptions())

18 sklearn/externals/joblib/numpy_pickle.py
View file
@@ -8,16 +8,18 @@

import pickle
import traceback
import sys, os
import sys
import os

if sys.version_info[0] == 3:
from pickle import _Unpickler as Unpickler
else:
from pickle import Unpickler

################################################################################
###############################################################################
# Utility objects for persistence.


class NDArrayWrapper(object):
""" An object to be persisted instead of numpy arrays.
@@ -28,7 +30,7 @@ def __init__(self, filename):
self.filename = filename


################################################################################
###############################################################################
# Pickler classes

class NumpyPickler(pickle.Pickler):
@@ -57,7 +59,7 @@ def save(self, obj):
self._npy_counter += 1
try:
filename = '%s_%02i.npy' % (self._filename,
self._npy_counter )
self._npy_counter)
self._filenames.append(filename)
self.np.save(filename, obj)
obj = NDArrayWrapper(os.path.basename(filename))
@@ -70,7 +72,6 @@ def save(self, obj):
pickle.Pickler.save(self, obj)



class NumpyUnpickler(Unpickler):
""" A subclass of the Unpickler to unpickle our numpy pickles.
"""
@@ -79,13 +80,12 @@ class NumpyUnpickler(Unpickler):
def __init__(self, filename, mmap_mode=None):
self._filename = filename
self.mmap_mode = mmap_mode
self._dirname = os.path.dirname(filename)
self._dirname = os.path.dirname(filename)
self.file = open(filename, 'rb')
Unpickler.__init__(self, self.file)
import numpy as np
self.np = np


def load_build(self):
""" This method is called to set the state of a knewly created
object.
@@ -107,12 +107,11 @@ def load_build(self):
nd_array_wrapper.filename))
self.stack.append(array)


# Be careful to register our new method.
dispatch[pickle.BUILD] = load_build


################################################################################
###############################################################################
# Utility functions

def dump(value, filename):
@@ -153,4 +152,3 @@ def load(filename, mmap_mode=None):
if 'unpickler' in locals() and hasattr(unpickler, 'file'):
unpickler.file.close()
return obj

70 sklearn/externals/joblib/parallel.py
View file
@@ -30,7 +30,8 @@
from .logger import Logger, short_format_time
from .my_exceptions import TransportableException, _mk_exception

################################################################################

###############################################################################
# CPU that works also when multiprocessing is not installed (python2.5)
def cpu_count():
""" Return the number of CPUs.
@@ -40,28 +41,24 @@ def cpu_count():
return multiprocessing.cpu_count()




################################################################################
###############################################################################
class WorkerInterrupt(Exception):
""" An exception that is not KeyboardInterrupt to allow subprocesses
to be interrupted.
"""
pass


################################################################################
###############################################################################
class SafeFunction(object):
""" Wraps a function to make it exception with full traceback in
their representation.
Useful for parallel computing with multiprocessing, for which
exceptions cannot be captured.
"""

def __init__(self, func):
self.func = func


def __call__(self, *args, **kwargs):
try:
return self.func(*args, **kwargs)
@@ -77,7 +74,7 @@ def __call__(self, *args, **kwargs):
raise TransportableException(text, e_type)


################################################################################
###############################################################################
def delayed(function):
""" Decorator used to capture the arguments of a function.
"""
@@ -94,20 +91,20 @@ def delayed_function(*args, **kwargs):
return delayed_function


################################################################################
###############################################################################
class ImmediateApply(object):
""" A non-delayed apply function.
"""
def __init__ (self, func, args, kwargs):
def __init__(self, func, args, kwargs):
# Don't delay the application, to avoid keeping the input
# arguments in memory
self.results = func(*args, **kwargs)

def get (self):
def get(self):
return self.results


################################################################################
###############################################################################
class CallBack(object):
""" Callback used by parallel: it is used for progress reporting, and
to add data to be processed
@@ -126,15 +123,15 @@ def print_progress(self):
# XXX: Not using the logger framework: need to
# learn to use logger better.
n_jobs = len(self.parallel._pool._pool)
if self.parallel.n_dispatched > 2*n_jobs:
if self.parallel.n_dispatched > 2 * n_jobs:
# Report less often
if not self.index % n_jobs == 0:
return
elapsed_time = time.time() - self.parallel._start_time
remaining_time = (elapsed_time/(self.index + 1)*
remaining_time = (elapsed_time / (self.index + 1) *
(self.parallel.n_dispatched - self.index - 1.))
if self.parallel._iterable:
# The object is still building it's job list
# The object is still building its job list
total = "%3i+" % self.parallel.n_dispatched
else:
total = "%3i " % self.parallel.n_dispatched
@@ -145,14 +142,14 @@ def print_progress(self):
writer = sys.stdout.write
writer('[%s]: Done %3i out of %s |elapsed: %s remaining: %s\n'
% (self.parallel,
self.index+1,
self.index + 1,
total,
short_format_time(elapsed_time),
short_format_time(remaining_time),
))


################################################################################
###############################################################################
class Parallel(Logger):
''' Helper class for readable parallel mapping.
@@ -283,15 +280,13 @@ class Parallel(Logger):
'''
def __init__(self, n_jobs=None, verbose=0, pre_dispatch='all'):
self.verbose = verbose
self.n_jobs = n_jobs
self.pre_dispatch = pre_dispatch
self._pool = None
# Not starting the pool in the __init__ is a design decision, to
# be able to close it ASAP, and not burden the user with closing
# it.
self.n_jobs = n_jobs
self.pre_dispatch = pre_dispatch
self._pool = None
# Not starting the pool in the __init__ is a design decision, to be
# able to close it ASAP, and not burden the user with closing it.
self._output = None
self._jobs = list()

self._jobs = list()

def dispatch(self, func, args, kwargs):
""" Queue the function for computing, with or without multiprocessing
@@ -300,7 +295,7 @@ def dispatch(self, func, args, kwargs):
job = ImmediateApply(func, args, kwargs)
if self.verbose:
print '[%s]: Done job %3i | elapsed: %s' % (
self, len(self._jobs)+1,
self, len(self._jobs) + 1,
short_format_time(time.time() - self._start_time)
)
self._jobs.append(job)
@@ -318,7 +313,6 @@ def dispatch(self, func, args, kwargs):
finally:
self._lock.release()


def dispatch_next(self):
""" Dispatch more data for parallel processing
"""
@@ -338,7 +332,6 @@ def dispatch_next(self):
self._iterable = None
return


def retrieve(self):
self._output = list()
while self._jobs:
@@ -361,13 +354,10 @@ def retrieve(self):
self._pool.terminate()
raise exception
elif isinstance(exception, TransportableException):
# Capture exception to add information on
# the local stack in addition to the distant
# stack
this_report = format_outer_frames(
context=10,
stack_start=1,
)
# Capture exception to add information on the local stack
# in addition to the distant stack
this_report = format_outer_frames(context=10,
stack_start=1)
report = """Multiprocessing exception:
%s
---------------------------------------------------------------------------
@@ -382,7 +372,6 @@ def retrieve(self):
raise exception_type(report)
raise exception


def __call__(self, iterable):
if self._jobs:
raise ValueError('This Parallel instance is already running')
@@ -429,12 +418,5 @@ def __call__(self, iterable):
self._output = None
return output


def __repr__(self):
return '%s(n_jobs=%s)' % (
self.__class__.__name__,
self.n_jobs,
)



return '%s(n_jobs=%s)' % (self.__class__.__name__, self.n_jobs)
1 sklearn/externals/joblib/test/__init__.py
View file
@@ -1 +0,0 @@

3 sklearn/externals/joblib/test/common.py
View file
@@ -6,6 +6,7 @@
# A decorator to run tests only when numpy is available
try:
import numpy as np

def with_numpy(func):
""" A decorator to skip tests requiring numpy.
"""
@@ -19,5 +20,3 @@ def my_func():
raise nose.SkipTest('Test requires numpy')
return my_func
np = None


2 sklearn/externals/joblib/test/test_format_stack.py
View file
@@ -11,7 +11,7 @@
from ..format_stack import safe_repr


################################################################################
###############################################################################

class Vicious(object):
def __repr__(self):
37 sklearn/externals/joblib/test/test_func_inspect.py
View file
@@ -13,41 +13,51 @@
from ..func_inspect import filter_args, get_func_name, get_func_code
from ..memory import Memory

################################################################################

###############################################################################
# Module-level functions, for tests
def f(x, y=0):
pass


def f2(x):
pass


# Create a Memory object to test decorated functions.
# We should be careful not to call the decorated functions, so that
# cache directories are not created in the temp dir.
mem = Memory(cachedir=tempfile.gettempdir())


@mem.cache
def g(x):
return x


def h(x, y=0, *args, **kwargs):
pass


def i(x=1):
pass


def j(x, y, **kwargs):
pass


def k(*args, **kwargs):
pass


class Klass(object):

def f(self, x):
return x

################################################################################

###############################################################################
# Tests

def test_filter_args():
@@ -56,8 +66,8 @@ def test_filter_args():
yield nose.tools.assert_equal, filter_args(f, ['y'], 0), {'x': 0}
yield nose.tools.assert_equal, filter_args(f, ['y'], 0, y=1), {'x': 0}
yield nose.tools.assert_equal, filter_args(f, ['x', 'y'], 0), {}
yield nose.tools.assert_equal, filter_args(f, [], 0, y=1), {'x':0, 'y':1}
yield nose.tools.assert_equal, filter_args(f, ['y'], x=2, y=1), {'x':2}
yield nose.tools.assert_equal, filter_args(f, [], 0, y=1), {'x': 0, 'y': 1}
yield nose.tools.assert_equal, filter_args(f, ['y'], x=2, y=1), {'x': 2}

yield nose.tools.assert_equal, filter_args(i, [], 2), {'x': 2}
yield nose.tools.assert_equal, filter_args(f2, [], x=1), {'x': 1}
@@ -71,35 +81,35 @@ def test_filter_args_method():

def test_filter_varargs():
yield nose.tools.assert_equal, filter_args(h, [], 1), \
{'x': 1, 'y': 0, '*':[], '**':{}}
{'x': 1, 'y': 0, '*': [], '**': {}}
yield nose.tools.assert_equal, filter_args(h, [], 1, 2, 3, 4), \
{'x': 1, 'y': 2, '*':[3, 4], '**':{}}
{'x': 1, 'y': 2, '*': [3, 4], '**': {}}
yield nose.tools.assert_equal, filter_args(h, [], 1, 25, ee=2), \
{'x': 1, 'y': 25, '*':[], '**':{'ee':2}}
{'x': 1, 'y': 25, '*': [], '**': {'ee': 2}}
yield nose.tools.assert_equal, filter_args(h, ['*'], 1, 2, 25, ee=2), \
{'x': 1, 'y': 2, '**':{'ee':2}}
{'x': 1, 'y': 2, '**': {'ee': 2}}


def test_filter_kwargs():
nose.tools.assert_equal(filter_args(k, [], 1, 2, ee=2),
{'*': [1, 2], '**':{'ee':2}})
{'*': [1, 2], '**': {'ee': 2}})
nose.tools.assert_equal(filter_args(k, [], 3, 4),
{'*': [3, 4], '**':{}})
{'*': [3, 4], '**': {}})


def test_filter_args_2():
nose.tools.assert_equal(filter_args(j, [], 1, 2, ee=2),
{'x': 1, 'y': 2, '**':{'ee':2}})
{'x': 1, 'y': 2, '**': {'ee': 2}})

nose.tools.assert_raises(ValueError, filter_args, f, 'a', None)
# Check that we capture an undefined argument
nose.tools.assert_raises(ValueError, filter_args, f, ['a'], None)
ff = functools.partial(f, 1)
# filter_args has to special-case partial
nose.tools.assert_equal(filter_args(ff, [], 1),
{'*': [1], '**':{}})
{'*': [1], '**': {}})
nose.tools.assert_equal(filter_args(ff, ['y'], 1),
{'*': [1], '**':{}})
{'*': [1], '**': {}})


def test_func_name():
@@ -126,7 +136,6 @@ def test_func_inspect_errors():
__file__.replace('.pyc', '.py'))



def test_bound_methods():
""" Make sure that calling the same method on two different instances
of the same class does resolv to different signatures.
10 sklearn/externals/joblib/test/test_logger.py
View file
@@ -17,10 +17,12 @@

from ..logger import PrintTime

################################################################################

###############################################################################
# Test fixtures
env = dict()


def setup():
""" Test setup.
"""
@@ -37,7 +39,7 @@ def teardown():
shutil.rmtree(env['dir'])


################################################################################
###############################################################################
# Tests
def test_print_time():
""" A simple smoke test for PrintTime.
@@ -55,10 +57,10 @@ def test_print_time():
print_time('Foo')
printed_text = sys.stderr.getvalue()
# Use regexps to be robust to time variations
match = r"Foo: 0\..s, 0\.0min\nFoo: 0\..s, 0.0min\nFoo: .\..s, 0.0min\n"
match = r"Foo: 0\..s, 0\.0min\nFoo: 0\..s, 0.0min\nFoo: " + \
r".\..s, 0.0min\n"
if not re.match(match, printed_text):
raise AssertionError('Excepted %s, got %s' %
(match, printed_text))
finally:
sys.stderr = orig_stderr

45 sklearn/externals/joblib/test/test_memory.py
View file
@@ -19,18 +19,20 @@
from ..memory import Memory, MemorizedFunc
from .common import with_numpy, np

################################################################################

###############################################################################
# Module-level variables for the tests
def f(x, y=1):
""" A module-level function for testing purposes.
"""
return x**2 + y
return x ** 2 + y


################################################################################
###############################################################################
# Test fixtures
env = dict()


def setup_module():
""" Test setup.
"""
@@ -39,29 +41,30 @@ def setup_module():
env['dir'] = cachedir
if os.path.exists(cachedir):
shutil.rmtree(cachedir)
# Don't make the cachedir, Memory should be able to do that on the
# fly
print 80*'_'
# Don't make the cachedir, Memory should be able to do that on the fly
print 80 * '_'
print 'test_memory setup'
print 80*'_'
print 80 * '_'


def _rmtree_onerror(func, path, excinfo):
print '!'*79
print '!' * 79
print 'os function failed:', repr(func)
print 'file to be removed:', path
print 'exception was:', excinfo[1]
print '!'*79
print '!' * 79


def teardown_module():
""" Test teardown.
"""
shutil.rmtree(env['dir'], False, _rmtree_onerror)
print 80*'_'
print 80 * '_'
print 'test_memory teardown'
print 80*'_'
print 80 * '_'


################################################################################
###############################################################################
# Helper function for the tests
def check_identity_lazy(func, accumulator):
""" Given a function and an accumulator (a list that grows every
@@ -79,7 +82,7 @@ def check_identity_lazy(func, accumulator):
yield nose.tools.assert_equal, len(accumulator), i + 1


################################################################################
###############################################################################
# Tests
def test_memory_integration():
""" Simple test of memory lazy evaluation.
@@ -88,6 +91,7 @@ def test_memory_integration():
# Rmk: this function has the same name than a module-level function,
# thus it serves as a test to see that both are identified
# as different.

def f(l):
accumulator.append(1)
return l
@@ -117,12 +121,13 @@ def f(l):


def test_no_memory():
""" Test memory with cachedir=None: no memoize
"""
""" Test memory with cachedir=None: no memoize """
accumulator = list()

def ff(l):
accumulator.append(1)
return l

mem = Memory(cachedir=None, verbose=0)
gg = mem.cache(ff)
for _ in range(4):
@@ -135,6 +140,7 @@ def ff(l):
def test_memory_kwarg():
" Test memory with a function with keyword arguments."
accumulator = list()

def g(l=None, m=1):
accumulator.append(1)
return l
@@ -151,6 +157,7 @@ def g(l=None, m=1):
def test_memory_lambda():
" Test memory with a function with a lambda."
accumulator = list()

def helper(x):
""" A helper function to define l as a lambda.
"""
@@ -202,7 +209,7 @@ def test_memory_warning_lambda_collisions():
memory = Memory(cachedir=env['dir'], verbose=0)
a = lambda x: x
a = memory.cache(a)
b = lambda x: x+1
b = lambda x: x + 1
b = memory.cache(b)

if not hasattr(warnings, 'catch_warnings'):
@@ -248,6 +255,7 @@ def test_memory_warning_collision_detection():
def test_memory_partial():
" Test memory with functools.partial."
accumulator = list()

def func(x, y):
""" A helper function to define l as a lambda.
"""
@@ -280,6 +288,7 @@ def count_and_append(x=[]):
x.append(None)
return len_x


def test_argument_change():
""" Check that if a function has a side effect in its arguments, it
should use the hash of changing arguments.
@@ -300,6 +309,7 @@ def test_memory_numpy():
# Check with memmapping and without.
for mmap_mode in (None, 'r'):
accumulator = list()

def n(l=None):
accumulator.append(1)
return l
@@ -319,6 +329,7 @@ def test_memory_exception():
""" Smoketest the exception handling of Memory.
"""
memory = Memory(cachedir=env['dir'], verbose=0)

class MyException(Exception):
pass

@@ -383,7 +394,6 @@ def test_func_dir():
yield nose.tools.assert_equal, a, g(1)



def test_persistence():
""" Test the memorized functions can be pickled and restored.
"""
@@ -415,4 +425,3 @@ def test_format_signature():
def test_format_signature_numpy():
""" Test the format signature formatting with numpy.
"""

2 sklearn/externals/joblib/test/test_my_exceptions.py
View file
@@ -5,10 +5,10 @@

from .. import my_exceptions


def test_inheritance():
assert_true(isinstance(my_exceptions.JoblibNameError(), NameError))
assert_true(isinstance(my_exceptions.JoblibNameError(),
my_exceptions.JoblibException))
assert_true(my_exceptions.JoblibNameError is
my_exceptions._mk_exception(NameError)[0])

82 sklearn/externals/joblib/test/test_numpy_pickle.py
View file
@@ -16,55 +16,83 @@
# filenames instead of open files as arguments.
from .. import numpy_pickle

################################################################################
###############################################################################
# Define a list of standard types.
# Borrowed from dill, initial author: Micheal McKerns:
# http://dev.danse.us/trac/pathos/browser/dill/dill_test2.py

typelist = []

# testing types
_none = None; typelist.append(_none)
_type = type; typelist.append(_type)
_bool = bool(1); typelist.append(_bool)
_int = int(1); typelist.append(_int)
_long = long(1); typelist.append(_long)
_float = float(1); typelist.append(_float)
_complex = complex(1); typelist.append(_complex)
_string = str(1); typelist.append(_string)
_unicode = unicode(1); typelist.append(_unicode)
_tuple = (); typelist.append(_tuple)
_list = []; typelist.append(_list)
_dict = {}; typelist.append(_dict)
_file = file; typelist.append(_file)
_buffer = buffer; typelist.append(_buffer)
_builtin = len; typelist.append(_builtin)
_none = None
typelist.append(_none)
_type = type
typelist.append(_type)
_bool = bool(1)
typelist.append(_bool)
_int = int(1)
typelist.append(_int)
_long = long(1)
typelist.append(_long)
_float = float(1)
typelist.append(_float)
_complex = complex(1)
typelist.append(_complex)
_string = str(1)
typelist.append(_string)
_unicode = unicode(1)
typelist.append(_unicode)
_tuple = ()
typelist.append(_tuple)
_list = []
typelist.append(_list)
_dict = {}
typelist.append(_dict)
_file = file
typelist.append(_file)
_buffer = buffer
typelist.append(_buffer)
_builtin = len
typelist.append(_builtin)


def _function(x):
yield x


class _class:
def _method(self):
pass


class _newclass(object):
def _method(self):
pass


typelist.append(_function)
typelist.append(_class)
typelist.append(_newclass) # <type 'type'>
_instance = _class(); typelist.append(_instance)
_object = _newclass(); typelist.append(_object) # <type 'class'>
def _function(x): yield x; typelist.append(_function)
typelist.append(_newclass) # <type 'type'>
_instance = _class()
typelist.append(_instance)
_object = _newclass()
typelist.append(_object) # <type 'class'>


################################################################################
###############################################################################
# Test fixtures

env = dict()


def setup_module():
""" Test setup.
"""
env['dir'] = mkdtemp()
env['filename'] = os.path.join(env['dir'], 'test.pkl')
print 80*'_'
print 80 * '_'
print 'setup numpy_pickle'
print 80*'_'
print 80 * '_'


def teardown_module():
@@ -73,12 +101,12 @@ def teardown_module():
shutil.rmtree(env['dir'])
#del env['dir']
#del env['filename']
print 80*'_'
print 80 * '_'
print 'teardown numpy_pickle'
print 80*'_'
print 80 * '_'


################################################################################
###############################################################################
# Tests

def test_standard_types():
@@ -137,5 +165,3 @@ def test_masked_array_persistence():
numpy_pickle.dump(a, filename)
b = numpy_pickle.load(filename, mmap_mode='r')
nose.tools.assert_true, isinstance(b, np.ma.masked_array)


26 sklearn/externals/joblib/test/test_parallel.py
View file
@@ -20,34 +20,41 @@

import nose

################################################################################

###############################################################################

def division(x, y):
return x/y
return x / y


def square(x):
return x**2
return x ** 2


def exception_raiser(x):
if x == 7:
raise ValueError
return x


def interrupt_raiser(x):
time.sleep(.05)
raise KeyboardInterrupt


def f(x, y=0, z=0):
""" A module-level function so that it can be spawn with
multiprocessing.
"""
return x**2 + y + z
return x ** 2 + y + z

################################################################################

###############################################################################
def test_cpu_count():
assert cpu_count() > 0

################################################################################

###############################################################################
# Test parallel
def test_simple_parallel():
X = range(10)
@@ -72,7 +79,7 @@ def test_parallel_pickling():
that cannot be pickled.
"""
def g(x):
return x**2
return x ** 2
nose.tools.assert_raises(PickleError,
Parallel(),
(delayed(g)(x) for x in range(10))
@@ -129,6 +136,7 @@ def test_dispatch_one_job():
""" Test that with only one job, Parallel does act as a iterator.
"""
queue = list()

def producer():
for i in range(6):
queue.append('Produced %i' % i)
@@ -154,6 +162,7 @@ def test_dispatch_multiprocessing():
return
manager = multiprocessing.Manager()
queue = manager.list()

def producer():
for i in range(6):
queue.append('Produced %i' % i)
@@ -176,7 +185,7 @@ def test_exception_dispatch():
)


################################################################################
###############################################################################
# Test helpers
def test_joblib_exception():
# Smoke-test the custom exception
@@ -190,4 +199,3 @@ def test_joblib_exception():
def test_safe_function():
safe_division = SafeFunction(division)
nose.tools.assert_raises(JoblibException, safe_division, 1, 0)

5 sklearn/externals/joblib/testing.py
View file
@@ -6,13 +6,14 @@
import warnings
import os.path


def warnings_to_stdout():
""" Redirect all warnings to stdout.
"""
showwarning_orig = warnings.showwarning

def showwarning(msg, cat, fname, lno, file=None, line=0):
showwarning_orig(msg, cat, os.path.basename(fname), line, sys.stdout)

warnings.showwarning = showwarning
#warnings.simplefilter('always')