Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch '2.7'

Conflicts:
	CHANGES.txt
	README.rst
	billiard/__init__.py
	billiard/pool.py
	billiard/util.py
  • Loading branch information...
commit 401ec585183a2ff75e8e1f32556abbf2a71fe649 2 parents 56978d6 + b1ddce7
@ask ask authored
View
5 .gitignore
@@ -17,3 +17,8 @@ devdatabase.db
bundle_version.gen
celeryd.log
celeryd.pid
+nosetests.xml
+coverage.xml
+cover/
+*.so
+.tox/
View
7 .travis.yml
@@ -0,0 +1,7 @@
+language: python
+python:
+ - 2.6
+ - 2.7
+install:
+ - pip install --use-mirrors tox
+script: TOXENV=py$(echo $TRAVIS_PYTHON_VERSION | tr -d .) tox
View
10 CHANGES.txt
@@ -5,6 +5,16 @@
- No longer compatible with Python 2.5
+2.7.3.28 - 2013-04-16
+---------------------
+
+- Pool: Fixed regression that disabled the deadlock
+ fix in 2.7.3.24
+
+- Pool: RestartFreqExceeded could be raised prematurely.
+
+- Process: Include pid in startup and process INFO logs.
+
2.7.3.27 - 2013-04-12
---------------------
View
2  MANIFEST.in
@@ -5,3 +5,5 @@ include Makefile
recursive-include Lib *.py
recursive-include Modules *.c *.h
recursive-include Doc *.rst *.py
+recursive-include funtests *.py
+recursive-include requirements *.txt
View
29 billiard/common.py
@@ -39,7 +39,7 @@ def reset_signals(handler=_shutdown_cleanup):
try:
signum = getattr(signal, sig)
current = signal.getsignal(signum)
- if current and current != signal.SIG_IGN:
+ if current is not None and current != signal.SIG_IGN:
signal.signal(signum, handler)
except (OSError, AttributeError, ValueError, RuntimeError):
pass
@@ -52,17 +52,22 @@ def __init__(self, maxR, maxT):
self.maxR, self.maxT = maxR, maxT
self.R, self.T = 0, None
- def step(self):
- now = time()
+ def step(self, now=None):
+ now = time() if now is None else now
R = self.R
if self.T and now - self.T >= self.maxT:
- self.R = 0
- elif self.maxR and R >= self.maxR:
- # verify that R has a value as it may have been reset
- # by another thread, and we want to avoid locking.
- if self.R:
- raise self.RestartFreqExceeded(
- "%r in %rs" % (self.R, self.maxT),
- )
+ # maxT passed, reset counter and time passed.
+ self.T, self.R = now, 0
+ elif self.maxR and self.R >= self.maxR:
+ # verify that R has a value as the result handler
+ # resets this when a job is accepted. If a job is accepted
+ # the startup probably went fine (startup restart burst
+ # protection)
+ if self.R: # pragma: no cover
+ pass
+ self.R = 0 # reset in case someone catches the error
+ raise self.RestartFreqExceeded("%r in %rs" % (R, self.maxT))
+ # first run sets T
+ if self.T is None:
+ self.T = now
self.R += 1
- self.T = now
View
12 billiard/pool.py
@@ -807,8 +807,6 @@ def __init__(self, processes=None, initializer=None, initargs=(),
self._initializer = initializer
self._initargs = initargs
self.lost_worker_timeout = lost_worker_timeout or LOST_WORKER_TIMEOUT
- self.max_restarts = max_restarts or round(processes * 100)
- self.restart_state = restart_state(max_restarts, max_restart_freq or 1)
self.on_process_up = on_process_up
self.on_process_down = on_process_down
self.on_timeout_set = on_timeout_set
@@ -833,6 +831,8 @@ def __init__(self, processes=None, initializer=None, initargs=(),
except NotImplementedError:
processes = 1
self._processes = processes
+ self.max_restarts = max_restarts or round(processes * 100)
+ self.restart_state = restart_state(max_restarts, max_restart_freq or 1)
if initializer is not None and \
not isinstance(initializer, collections.Callable):
@@ -976,8 +976,6 @@ def _join_exited_workers(self, shutdown=False):
exitcodes[worker_pid])
break
for worker in values(cleaned):
- if self._putlock is not None:
- self._putlock.release()
if self.on_process_down:
self.on_process_down(worker)
return list(exitcodes.values())
@@ -1043,7 +1041,11 @@ def did_start_ok(self):
def _maintain_pool(self):
""""Clean up any exited workers and start replacements for them.
"""
- self._repopulate_pool(self._join_exited_workers())
+ joined = self._join_exited_workers()
+ self._repopulate_pool(joined)
+ for i in range(len(joined)):
+ if self._putlock is not None:
+ self._putlock.release()
def maintain_pool(self, *args, **kwargs):
if self._worker_handler._state == RUN and self._state == RUN:
View
16 billiard/process.py
@@ -46,9 +46,10 @@ def current_process():
def _cleanup():
# check for processes which have finished
- for p in list(_current_process._children):
- if p._popen.poll() is not None:
- _current_process._children.discard(p)
+ if _current_process is not None:
+ for p in list(_current_process._children):
+ if p._popen.poll() is not None:
+ _current_process._children.discard(p)
def active_children(_cleanup=_cleanup):
@@ -60,7 +61,9 @@ def active_children(_cleanup=_cleanup):
except TypeError:
# called after gc collect so _cleanup does not exist anymore
return []
- return list(_current_process._children)
+ if _current_process is not None:
+ return list(_current_process._children)
+ return []
class Process(object):
@@ -250,7 +253,7 @@ def _bootstrap(self):
# delay finalization of the old process object until after
# _run_after_forkers() is executed
del old_process
- util.info('child process calling self.run()')
+ util.info('child process %s calling self.run()', self.pid)
try:
self.run()
exitcode = 0
@@ -272,7 +275,8 @@ def _bootstrap(self):
sys.stderr.write('Process %s:\n' % self.name)
traceback.print_exc()
finally:
- util.info('process exiting with exitcode %d', exitcode)
+ util.info('process %s exiting with exitcode %d',
+ self.pid, exitcode)
sys.stdout.flush()
sys.stderr.flush()
return exitcode
View
18 billiard/tests/__init__.py
@@ -0,0 +1,18 @@
+from __future__ import absolute_import
+
+import atexit
+
+
+def teardown():
+ # Workaround for multiprocessing bug where logging
+ # is attempted after global already collected at shutdown.
+ cancelled = set()
+ try:
+ import multiprocessing.util
+ cancelled.add(multiprocessing.util._exit_function)
+ except (AttributeError, ImportError):
+ pass
+
+ atexit._exithandlers[:] = [
+ e for e in atexit._exithandlers if e[0] not in cancelled
+ ]
View
85 billiard/tests/compat.py
@@ -0,0 +1,85 @@
+from __future__ import absolute_import
+
+import sys
+
+
+class WarningMessage(object):
+
+ """Holds the result of a single showwarning() call."""
+
+ _WARNING_DETAILS = ('message', 'category', 'filename', 'lineno', 'file',
+ 'line')
+
+ def __init__(self, message, category, filename, lineno, file=None,
+ line=None):
+ local_values = locals()
+ for attr in self._WARNING_DETAILS:
+ setattr(self, attr, local_values[attr])
+
+ self._category_name = category and category.__name__ or None
+
+ def __str__(self):
+ return ('{message : %r, category : %r, filename : %r, lineno : %s, '
+ 'line : %r}' % (self.message, self._category_name,
+ self.filename, self.lineno, self.line))
+
+
+class catch_warnings(object):
+
+ """A context manager that copies and restores the warnings filter upon
+ exiting the context.
+
+ The 'record' argument specifies whether warnings should be captured by a
+ custom implementation of warnings.showwarning() and be appended to a list
+ returned by the context manager. Otherwise None is returned by the context
+ manager. The objects appended to the list are arguments whose attributes
+ mirror the arguments to showwarning().
+
+ The 'module' argument is to specify an alternative module to the module
+ named 'warnings' and imported under that name. This argument is only
+ useful when testing the warnings module itself.
+
+ """
+
+ def __init__(self, record=False, module=None):
+ """Specify whether to record warnings and if an alternative module
+ should be used other than sys.modules['warnings'].
+
+ For compatibility with Python 3.0, please consider all arguments to be
+ keyword-only.
+
+ """
+ self._record = record
+ self._module = module is None and sys.modules['warnings'] or module
+ self._entered = False
+
+ def __repr__(self):
+ args = []
+ if self._record:
+ args.append('record=True')
+ if self._module is not sys.modules['warnings']:
+ args.append('module=%r' % self._module)
+ name = type(self).__name__
+ return '%s(%s)' % (name, ', '.join(args))
+
+ def __enter__(self):
+ if self._entered:
+ raise RuntimeError('Cannot enter %r twice' % self)
+ self._entered = True
+ self._filters = self._module.filters
+ self._module.filters = self._filters[:]
+ self._showwarning = self._module.showwarning
+ if self._record:
+ log = []
+
+ def showwarning(*args, **kwargs):
+ log.append(WarningMessage(*args, **kwargs))
+
+ self._module.showwarning = showwarning
+ return log
+
+ def __exit__(self, *exc_info):
+ if not self._entered:
+ raise RuntimeError('Cannot exit %r without entering first' % self)
+ self._module.filters = self._filters
+ self._module.showwarning = self._showwarning
View
98 billiard/tests/test_common.py
@@ -0,0 +1,98 @@
+from __future__ import absolute_import
+from __future__ import with_statement
+
+import os
+import signal
+
+from contextlib import contextmanager
+from mock import call, patch, Mock
+from time import time
+
+from billiard.common import (
+ _shutdown_cleanup,
+ reset_signals,
+ restart_state,
+)
+
+from .utils import Case
+
+
+def signo(name):
+ return getattr(signal, name)
+
+
+@contextmanager
+def termsigs(*sigs):
+ from billiard import common
+ prev, common.TERMSIGS = common.TERMSIGS, sigs
+ try:
+ yield
+ finally:
+ common.TERMSIGS = prev
+
+
+class test_reset_signals(Case):
+
+ def test_shutdown_handler(self):
+ with patch('sys.exit') as exit:
+ _shutdown_cleanup(15, Mock())
+ self.assertTrue(exit.called)
+ self.assertEqual(os.WTERMSIG(exit.call_args[0][0]), 15)
+
+ def test_does_not_reset_ignored_signal(self, sigs=['SIGTERM']):
+ with self.assert_context(sigs, signal.SIG_IGN) as (_, SET):
+ self.assertFalse(SET.called)
+
+ def test_does_not_reset_if_current_is_None(self, sigs=['SIGTERM']):
+ with self.assert_context(sigs, None) as (_, SET):
+ self.assertFalse(SET.called)
+
+ def test_resets_for_SIG_DFL(self, sigs=['SIGTERM', 'SIGINT', 'SIGUSR1']):
+ with self.assert_context(sigs, signal.SIG_DFL) as (_, SET):
+ SET.assert_has_calls([
+ call(signo(sig), _shutdown_cleanup) for sig in sigs
+ ])
+
+ def test_resets_for_obj(self, sigs=['SIGTERM', 'SIGINT', 'SIGUSR1']):
+ with self.assert_context(sigs, object()) as (_, SET):
+ SET.assert_has_calls([
+ call(signo(sig), _shutdown_cleanup) for sig in sigs
+ ])
+
+ def test_handles_errors(self, sigs=['SIGTERM']):
+ for exc in (OSError(), AttributeError(),
+ ValueError(), RuntimeError()):
+ with self.assert_context(sigs, signal.SIG_DFL, exc) as (_, SET):
+ self.assertTrue(SET.called)
+
+ @contextmanager
+ def assert_context(self, sigs, get_returns=None, set_effect=None):
+ with termsigs(*sigs):
+ with patch('signal.getsignal') as GET:
+ with patch('signal.signal') as SET:
+ GET.return_value = get_returns
+ SET.side_effect = set_effect
+ reset_signals()
+ GET.assert_has_calls([
+ call(signo(sig)) for sig in sigs
+ ])
+ yield GET, SET
+
+
+class test_restart_state(Case):
+
+ def test_raises(self):
+ s = restart_state(100, 1) # max 100 restarts in 1 second.
+ s.R = 99
+ s.step()
+ with self.assertRaises(s.RestartFreqExceeded):
+ s.step()
+
+ def test_time_passed_resets_counter(self):
+ s = restart_state(100, 10)
+ s.R, s.T = 100, time()
+ with self.assertRaises(s.RestartFreqExceeded):
+ s.step()
+ s.R, s.T = 100, time()
+ s.step(time() + 20)
+ self.assertEqual(s.R, 1)
View
12 billiard/tests/test_package.py
@@ -0,0 +1,12 @@
+from __future__ import absolute_import
+
+import billiard
+
+from .utils import Case
+
+
+class test_billiard(Case):
+
+ def test_has_version(self):
+ self.assertTrue(billiard.__version__)
+ self.assertIsInstance(billiard.__version__, str)
View
144 billiard/tests/utils.py
@@ -0,0 +1,144 @@
+from __future__ import absolute_import
+from __future__ import with_statement
+
+import re
+import sys
+import warnings
+
+try:
+ import unittest # noqa
+ unittest.skip
+ from unittest.util import safe_repr, unorderable_list_difference
+except AttributeError:
+ import unittest2 as unittest # noqa
+ from unittest2.util import safe_repr, unorderable_list_difference # noqa
+
+from .compat import catch_warnings
+
+# -- adds assertWarns from recent unittest2, not in Python 2.7.
+
+
+class _AssertRaisesBaseContext(object):
+
+ def __init__(self, expected, test_case, callable_obj=None,
+ expected_regex=None):
+ self.expected = expected
+ self.failureException = test_case.failureException
+ self.obj_name = None
+ if isinstance(expected_regex, basestring):
+ expected_regex = re.compile(expected_regex)
+ self.expected_regex = expected_regex
+
+
+class _AssertWarnsContext(_AssertRaisesBaseContext):
+ """A context manager used to implement TestCase.assertWarns* methods."""
+
+ def __enter__(self):
+ # The __warningregistry__'s need to be in a pristine state for tests
+ # to work properly.
+ warnings.resetwarnings()
+ for v in sys.modules.values():
+ if getattr(v, '__warningregistry__', None):
+ v.__warningregistry__ = {}
+ self.warnings_manager = catch_warnings(record=True)
+ self.warnings = self.warnings_manager.__enter__()
+ warnings.simplefilter('always', self.expected)
+ return self
+
+ def __exit__(self, exc_type, exc_value, tb):
+ self.warnings_manager.__exit__(exc_type, exc_value, tb)
+ if exc_type is not None:
+ # let unexpected exceptions pass through
+ return
+ try:
+ exc_name = self.expected.__name__
+ except AttributeError:
+ exc_name = str(self.expected)
+ first_matching = None
+ for m in self.warnings:
+ w = m.message
+ if not isinstance(w, self.expected):
+ continue
+ if first_matching is None:
+ first_matching = w
+ if (self.expected_regex is not None and
+ not self.expected_regex.search(str(w))):
+ continue
+ # store warning for later retrieval
+ self.warning = w
+ self.filename = m.filename
+ self.lineno = m.lineno
+ return
+ # Now we simply try to choose a helpful failure message
+ if first_matching is not None:
+ raise self.failureException(
+ '%r does not match %r' % (
+ self.expected_regex.pattern, str(first_matching)))
+ if self.obj_name:
+ raise self.failureException(
+ '%s not triggered by %s' % (exc_name, self.obj_name))
+ else:
+ raise self.failureException('%s not triggered' % exc_name)
+
+
+class Case(unittest.TestCase):
+
+ def assertWarns(self, expected_warning):
+ return _AssertWarnsContext(expected_warning, self, None)
+
+ def assertWarnsRegex(self, expected_warning, expected_regex):
+ return _AssertWarnsContext(expected_warning, self,
+ None, expected_regex)
+
+ def assertDictContainsSubset(self, expected, actual, msg=None):
+ missing, mismatched = [], []
+
+ for key, value in expected.iteritems():
+ if key not in actual:
+ missing.append(key)
+ elif value != actual[key]:
+ mismatched.append('%s, expected: %s, actual: %s' % (
+ safe_repr(key), safe_repr(value),
+ safe_repr(actual[key])))
+
+ if not (missing or mismatched):
+ return
+
+ standard_msg = ''
+ if missing:
+ standard_msg = 'Missing: %s' % ','.join(map(safe_repr, missing))
+
+ if mismatched:
+ if standard_msg:
+ standard_msg += '; '
+ standard_msg += 'Mismatched values: %s' % (
+ ','.join(mismatched))
+
+ self.fail(self._formatMessage(msg, standard_msg))
+
+ def assertItemsEqual(self, expected_seq, actual_seq, msg=None):
+ missing = unexpected = None
+ try:
+ expected = sorted(expected_seq)
+ actual = sorted(actual_seq)
+ except TypeError:
+ # Unsortable items (example: set(), complex(), ...)
+ expected = list(expected_seq)
+ actual = list(actual_seq)
+ missing, unexpected = unorderable_list_difference(
+ expected, actual)
+ else:
+ return self.assertSequenceEqual(expected, actual, msg=msg)
+
+ errors = []
+ if missing:
+ errors.append(
+ 'Expected, but missing:\n %s' % (safe_repr(missing), ),
+ )
+ if unexpected:
+ errors.append(
+ 'Unexpected, but present:\n %s' % (safe_repr(unexpected), ),
+ )
+ if errors:
+ standardMsg = '\n'.join(errors)
+ self.fail(self._formatMessage(msg, standardMsg))
View
9 billiard/util.py
@@ -257,7 +257,9 @@ def __repr__(self):
x += ', exitprority=' + str(self._key[0])
return x + '>'
- def _run_finalizers(minpriority=None): # noqa
+ def _run_finalizers(minpriority=None,
+ _finalizer_registry=_finalizer_registry,
+ sub_debug=sub_debug): # noqa
"""Run all finalizers whose exit priority is not None
and at least minpriority'.
@@ -292,11 +294,12 @@ def is_exiting(): # noqa
'''
return _exiting or _exiting is None
- def _exit_function(): # noqa
+ def _exit_function(info=info, debug=debug,
+ _run_finalizers=_run_finalizers,
+ active_children=active_children): # noqa
'''
Clean up on exit
'''
-
global _exiting
info('process shutting down')
View
5 funtests/__init__.py
@@ -0,0 +1,5 @@
+import os
+import sys
+
+sys.path.insert(0, os.pardir)
+sys.path.insert(0, os.getcwd())
View
4 funtests/setup.cfg
@@ -0,0 +1,4 @@
+[nosetests]
+verbosity = 1
+detailed-errors = 1
+where = tests
View
59 funtests/setup.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+try:
+ from setuptools import setup
+ from setuptools.command.install import install
+except ImportError:
+ from ez_setup import use_setuptools
+ use_setuptools()
+ from setuptools import setup # noqa
+ from setuptools.command.install import install # noqa
+
+
+class no_install(install):
+
+ def run(self, *args, **kwargs):
+ import sys
+ sys.stderr.write("""
+-------------------------------------------------------
+The billiard functional test suite cannot be installed.
+-------------------------------------------------------
+
+
+But you can execute the tests by running the command:
+
+ $ python setup.py test
+
+
+""")
+
+
+setup(
+ name='billiard-funtests',
+ version='DEV',
+ description='Functional test suite for billiard',
+ author='Ask Solem',
+ author_email='ask@celeryproject.org',
+ url='http://github.com/celery/billiard',
+ platforms=['any'],
+ packages=[],
+ data_files=[],
+ zip_safe=False,
+ cmdclass={'install': no_install},
+ test_suite='nose.collector',
+ build_requires=[
+ 'nose',
+ 'nose-cover3',
+ 'unittest2',
+ 'coverage>=3.0',
+ ],
+ classifiers=[
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python',
+ 'Programming Language :: C'
+ 'License :: OSI Approved :: BSD License',
+ 'Intended Audience :: Developers',
+ ],
+ long_description='Do not install this package',
+)
View
7 funtests/tests/__init__.py
@@ -0,0 +1,7 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.getcwd(), os.pardir))
+print(sys.path[0])
+sys.path.insert(0, os.getcwd())
+print(sys.path[0])
View
0  billiard/tests/test_multiprocessing.py → funtests/tests/test_multiprocessing.py
File renamed without changes
View
5 requirements/test-ci.txt
@@ -0,0 +1,5 @@
+coverage>=3.0
+redis
+pymongo
+SQLAlchemy
+PyOpenSSL
View
4 requirements/test.txt
@@ -0,0 +1,4 @@
+unittest2>=0.4.0
+nose
+nose-cover3
+mock
View
3  requirements/test3.txt
@@ -0,0 +1,3 @@
+nose
+nose-cover3
+mock
View
29 setup.py
@@ -166,6 +166,26 @@ def add_doc(m):
"""
long_description += open(os.path.join(HERE, 'CHANGES.txt')).read()
+# -*- Installation Requires -*-
+
+py_version = sys.version_info
+is_jython = sys.platform.startswith('java')
+is_pypy = hasattr(sys, 'pypy_version_info')
+
+
+def strip_comments(l):
+ return l.split('#', 1)[0].strip()
+
+
+def reqs(f):
+ return list(filter(None, [strip_comments(l) for l in open(
+ os.path.join(os.getcwd(), 'requirements', f)).readlines()]))
+
+if py_version[0] == 3:
+ tests_require = reqs('test3.txt')
+else:
+ tests_require = reqs('test.txt')
+
def run_setup(with_extensions=True):
extensions = []
@@ -194,7 +214,7 @@ def run_setup(with_extensions=True):
url=meta['homepage'],
zip_safe=False,
license='BSD',
- tests_require=['nose', 'nose-cover3'],
+ tests_require=tests_require,
test_suite='nose.collector',
classifiers=[
'Development Status :: 5 - Production/Stable',
@@ -223,6 +243,7 @@ def run_setup(with_extensions=True):
try:
run_setup(not (is_jython or is_pypy or is_py3k))
except BaseException:
- import traceback
- sys.stderr.write(BUILD_WARNING % '\n'.join(traceback.format_stack(), ))
- run_setup(False)
+ if 'test' not in sys.argv:
+ import traceback
+ sys.stderr.write(BUILD_WARNING % '\n'.join(traceback.format_stack(), ))
+ run_setup(False)
View
37 tox.ini
@@ -0,0 +1,37 @@
+[tox]
+envlist = py25,py26,py27
+
+[testenv]
+distribute = True
+sitepackages = False
+commands = nosetests
+
+[testenv:py27]
+basepython = python2.7
+deps = -r{toxinidir}/requirements/test.txt
+ -r{toxinidir}/requirements/test-ci.txt
+commands = nosetests --with-xunit \
+ --xunit-file={toxinidir}/nosetests.xml \
+ --with-coverage3 --cover3-xml \
+ --cover3-html-dir={toxinidir}/cover \
+ --cover3-xml-file={toxinidir}/coverage.xml
+
+[testenv:py26]
+basepython = python2.6
+deps = -r{toxinidir}/requirements/test.txt
+ -r{toxinidir}/requirements/test-ci.txt
+commands = nosetests --with-xunit \
+ --xunit-file={toxinidir}/nosetests.xml \
+ --with-coverage3 --cover3-xml \
+ --cover3-html-dir={toxinidir}/cover \
+ --cover3-xml-file={toxinidir}/coverage.xml
+
+[testenv:py25]
+basepython = python2.5
+deps = -r{toxinidir}/requirements/test.txt
+ -r{toxinidir}/requirements/test-ci.txt
+commands = nosetests --with-xunit \
+ --xunit-file={toxinidir}/nosetests.xml \
+ --with-coverage3 --cover3-xml \
+ --cover3-html-dir={toxinidir}/cover \
+ --cover3-xml-file={toxinidir}/coverage.xml
Please sign in to comment.
Something went wrong with that request. Please try again.