diff --git a/AUTHORS b/AUTHORS index 14b3a5bc..aced90a4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -40,3 +40,5 @@ Ordered by date of first contribution: WoLpH Darjus Loktevic Greg Taylor + Ionel Maries Cristian + Jonas Haag diff --git a/Changelog b/Changelog index 623cc38f..579397d1 100644 --- a/Changelog +++ b/Changelog @@ -10,24 +10,44 @@ 2.4.0 ===== :release-date: 2011-10-29 04:00 P.M BST +:by: Ask Solem + +.. _240-important: Important Notes --------------- -This release adds south_ migrations, which well assist users in automatically +This release adds `South`_ migrations, which well assist users in automatically updating their database schemas with each django-celery release. -.. _south: http://pypi.python.org/pypi/South/ +.. _`South`: http://pypi.python.org/pypi/South/ + +.. _240-news: News ---- * Now depends on Celery 2.4.0 or higher. -* South migrations have been added. Migration 0001 is a snapshot from the - last release, 2.3.3. For those who do not use south, no action is required. - south users will want to read the upgrading instructions in the - following section. +* South migrations have been added. + + Migration 0001 is a snapshot from the previous stable release (2.3.3). + For those who do not use South, no action is required. + South users will want to read the :ref:`240-upgrade-south` section + below. + + Contributed by Greg Taylor. + +* Test runner now compatible with Django 1.4. + + Test runners are now classes instead of functions, + so you have to change the ``TEST_RUNNER`` setting to read:: + + TEST_RUNNER = "djcelery.contrib.test_runner.CeleryTestSuiteRunner" + + Contributed by Jonas Haag. + +.. _240-upgrade_south: Upgrading for south users ------------------------- diff --git a/FAQ b/FAQ index b1f72057..3aa388cf 100644 --- a/FAQ +++ b/FAQ @@ -63,7 +63,7 @@ To use this test runner, add the following to your ``settings.py``: .. code-block:: python - TEST_RUNNER = "djcelery.tests.runners.run_tests", + TEST_RUNNER = "djcelery.tests.runners.CeleryTestSuiteRunner", TEST_APPS = ( "app1", "app2", @@ -76,6 +76,6 @@ Or, if you just want to skip the celery tests: .. code-block:: python INSTALLED_APPS = (.....) - TEST_RUNNER = "djcelery.tests.runners.run_tests", + TEST_RUNNER = "djcelery.tests.runners.CeleryTestSuiteRunner", TEST_APPS = filter(lambda k: k != "celery", INSTALLED_APPS) diff --git a/contrib/release/removepyc.sh b/contrib/release/removepyc.sh new file mode 100755 index 00000000..9aaf3658 --- /dev/null +++ b/contrib/release/removepyc.sh @@ -0,0 +1,3 @@ +#!/bin/bash +(cd "${1:-.}"; + find . -name "*.pyc" | xargs rm -- 2>/dev/null) || echo "ok" diff --git a/djcelery/admin.py b/djcelery/admin.py index efedcbfc..cf139b2d 100644 --- a/djcelery/admin.py +++ b/djcelery/admin.py @@ -49,7 +49,7 @@ def colored_state(task): return """%s""" % (color, state) -@display_field(_("state"), "last_timestamp") +@display_field(_("state"), "last_heartbeat") def node_state(node): state = node.is_alive() and "ONLINE" or "OFFLINE" color = NODE_STATE_COLORS[state] @@ -171,7 +171,6 @@ def rate_limit_tasks(self, request, queryset): "object_name": force_unicode(opts.verbose_name), "action_checkbox_name": helpers.ACTION_CHECKBOX_NAME, "opts": opts, - "root_path": self.admin_site.root_path, "app_label": app_label, } diff --git a/djcelery/contrib/test_runner.py b/djcelery/contrib/test_runner.py index 7e43f4f6..7ce0a856 100644 --- a/djcelery/contrib/test_runner.py +++ b/djcelery/contrib/test_runner.py @@ -1,23 +1,23 @@ from __future__ import absolute_import from django.conf import settings -from django.test.simple import run_tests as run_tests_orig +from django.test.simple import DjangoTestSuiteRunner USAGE = """\ Custom test runner to allow testing of celery delayed tasks. """ - -def run_tests(test_labels, *args, **kwargs): +class CeleryTestSuiteRunner(DjangoTestSuiteRunner): """Django test runner allowing testing of celery delayed tasks. All tasks are run locally, not in a worker. To use this runner set ``settings.TEST_RUNNER``:: - TEST_RUNNER = "celery.contrib.test_runner.run_tests" + TEST_RUNNER = "celery.contrib.test_runner.CeleryTestSuiteRunner" """ - settings.CELERY_ALWAYS_EAGER = True - settings.CELERY_EAGER_PROPAGATES_EXCEPTIONS = True # Issue #75 - return run_tests_orig(test_labels, *args, **kwargs) + def setup_test_environment(self, **kwargs): + super(CeleryTestSuiteRunner, self).setup_test_environment(**kwargs) + settings.CELERY_ALWAYS_EAGER = True + settings.CELERY_EAGER_PROPAGATES_EXCEPTIONS = True # Issue #75 diff --git a/djcelery/loaders.py b/djcelery/loaders.py index 92bd49cb..963aa5a7 100644 --- a/djcelery/loaders.py +++ b/djcelery/loaders.py @@ -86,13 +86,17 @@ def on_worker_init(self): warnings.warn("Using settings.DEBUG leads to a memory leak, never " "use this setting in production environments!") - # the parent process may have established these, - # so need to close them. self.close_database() self.close_cache() self.import_default_modules() autodiscover() + def on_worker_process_init(self): + # the parent process may have established these, + # so need to close them. + self.close_database() + self.close_cache() + def mail_admins(self, subject, body, fail_silently=False, **kwargs): return mail_admins(subject, body, fail_silently=fail_silently) diff --git a/djcelery/managers.py b/djcelery/managers.py index 59c6fc3b..4d405992 100644 --- a/djcelery/managers.py +++ b/djcelery/managers.py @@ -89,6 +89,12 @@ def connection_for_read(self): return connections[self.db] return connection + def current_engine(self): + try: + return settings.DATABASES[self.db]["ENGINE"] + except AttributeError: + return settings.DATABASE_ENGINE + class ResultManager(ExtendedManager): @@ -156,7 +162,7 @@ def store_result(self, task_id, result, status, traceback=None): "traceback": traceback}) def warn_if_repeatable_read(self): - if settings.DATABASE_ENGINE.lower() == "mysql": + if "mysql" in self.current_engine().lower(): cursor = self.connection_for_read().cursor() if cursor.execute("SELECT @@tx_isolation"): isolation = cursor.fetchone()[0] diff --git a/djcelery/tests/runners.py b/djcelery/tests/runners.py deleted file mode 100644 index 148b5c79..00000000 --- a/djcelery/tests/runners.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import absolute_import - -from django.conf import settings -from django.test.simple import run_tests as django_test_runner - - -def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=None, - **kwargs): - """ Test runner that only runs tests for the apps - listed in ``settings.TEST_APPS``. - """ - extra_tests = extra_tests or [] - app_labels = getattr(settings, "TEST_APPS", test_labels) - - # Seems to be deleting the test database file twice :( - from celery.utils import noop - from django.db import connection - connection.creation.destroy_test_db = noop - return django_test_runner(app_labels, - verbosity=verbosity, interactive=interactive, - extra_tests=extra_tests, **kwargs) diff --git a/docs/cookbook/unit-testing.rst b/docs/cookbook/unit-testing.rst index 17073d10..eeac1392 100644 --- a/docs/cookbook/unit-testing.rst +++ b/docs/cookbook/unit-testing.rst @@ -34,7 +34,7 @@ To enable the test runner, set the following settings: .. code-block:: python - TEST_RUNNER = 'djcelery.contrib.test_runner.run_tests' + TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner' Then we can put the tests in a ``tests.py`` somewhere: diff --git a/examples/clickcounter/__init__.py b/examples/clickcounter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/clickcounter/clickmuncher/__init__.py b/examples/clickcounter/clickmuncher/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/clickcounter/clickmuncher/actors.py b/examples/clickcounter/clickmuncher/actors.py new file mode 100644 index 00000000..68a67111 --- /dev/null +++ b/examples/clickcounter/clickmuncher/actors.py @@ -0,0 +1,62 @@ +from collections import defaultdict + +import cl + +from celery import current_app as celery +from celery.utils.timer2 import apply_interval + + +from clickmuncher import models + + +class Clicks(cl.Actor): + default_routing_key = "Clicks" + + class state: + model = models.Click + clicks = defaultdict(lambda: 0) + + def increment(self, url, clicks=1): + self.clicks[url] += clicks + + def flush(self): + print("FLUSH!!!") + print("STATE: %r" % (self.clicks, )) + for url, clicks in self.clicks.iteritems(): + self.model.objects.increment_clicks(url, clicks) + self.clicks.clear() + + def __init__(self, connection=None, *args, **kwargs): + if not connection: + connection = celery.broker_connection() + super(Clicks, self).__init__(connection, *args, **kwargs) + + def increment(self, url, clicks=1): + self.cast("increment", {"url": url, "clicks": clicks}) + + +class Agent(cl.Agent): + actors = [Clicks()] + flush_every = 5 + + def __init__(self, connection=None, *args, **kwargs): + if not connection: + connection = celery.broker_connection() + self.clicks = Clicks() + self.actors = [self.clicks] + self.timers = [] + super(Agent, self).__init__(connection, *args, **kwargs) + + def on_consume_ready(self, *args, **kwargs): + print("INSTALLING TIMER") + self.timers.append(apply_interval(self.flush_every * 1000, + self.clicks.state.flush)) + + def stop(self): + for entry in self.timers: + entry.cancel() + super(Agent, self).stop() + + +if __name__ == "__main__": + Agent().run_from_commandline() diff --git a/examples/clickcounter/clickmuncher/models.py b/examples/clickcounter/clickmuncher/models.py new file mode 100644 index 00000000..f6b2758f --- /dev/null +++ b/examples/clickcounter/clickmuncher/models.py @@ -0,0 +1,27 @@ +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from djcelery.managers import ExtendedManager + + +class ClickManager(ExtendedManager): + + def increment_clicks(self, url, increment=1): + """Increment the click count for an URL.""" + obj, created = self.get_or_create(url=url, + defaults={"clicks": increment}) + if not created: + obj.clicks += increment + obj.save() + + return obj.clicks + + +class Click(models.Model): + url = models.URLField(_(u"URL"), verify_exists=False, unique=True) + clicks = models.PositiveIntegerField(_(u"clicks"), default=0) + + objects = ClickManager() + + class Meta: + verbose_name = _(u"click") + verbose_name_plural = _(u"clicks") diff --git a/examples/clickcounter/manage.py b/examples/clickcounter/manage.py new file mode 100644 index 00000000..3e4eedc9 --- /dev/null +++ b/examples/clickcounter/manage.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +from django.core.management import execute_manager +import imp +try: + imp.find_module('settings') # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__) + sys.exit(1) + +import settings + +if __name__ == "__main__": + execute_manager(settings) diff --git a/examples/clickcounter/settings.py b/examples/clickcounter/settings.py new file mode 100644 index 00000000..4efbb6cc --- /dev/null +++ b/examples/clickcounter/settings.py @@ -0,0 +1,154 @@ +# Django settings for clickcounter project. + +import djcelery +djcelery.setup_loader() + +import os +import sys +sys.path.append(os.getcwd()) + +DEBUG = True +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + # ('Your Name', 'your_email@example.com'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'test.db', + 'USER': '', + 'PASSWORD': '', + 'HOST': '', + 'PORT': '', + } +} + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# On Unix systems, a value of None will cause Django to use the same +# timezone as the operating system. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'America/Chicago' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# If you set this to False, Django will not format dates, numbers and +# calendars according to the current locale +USE_L10N = True + +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/home/media/media.lawrence.com/media/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" +MEDIA_URL = '' + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/home/media/media.lawrence.com/static/" +STATIC_ROOT = '' + +# URL prefix for static files. +# Example: "http://media.lawrence.com/static/" +STATIC_URL = '/static/' + +# Additional locations of static files +STATICFILES_DIRS = ( + # Put strings here, like "/home/html/static" or "C:/www/django/static". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +# 'django.contrib.staticfiles.finders.DefaultStorageFinder', +) + +# Make this unique, and don't share it with anybody. +SECRET_KEY = '(up^8#vyl9fj+x4_o!5!-h-u-ydixwxb!pih)%5j#jsj2btir9' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', +# 'django.template.loaders.eggs.Loader', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + # Uncomment the next line for simple clickjacking protection: + # 'django.middleware.clickjacking.XFrameOptionsMiddleware', +) + +ROOT_URLCONF = 'clickcounter.urls' + +TEMPLATE_DIRS = ( + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'djcelery', + + 'clickmuncher', +) + +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + } + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins'], + 'level': 'ERROR', + 'propagate': True, + }, + } +} diff --git a/examples/clickcounter/urls.py b/examples/clickcounter/urls.py new file mode 100644 index 00000000..cd4563ad --- /dev/null +++ b/examples/clickcounter/urls.py @@ -0,0 +1,17 @@ +from django.conf.urls import patterns, include, url + +# Uncomment the next two lines to enable the admin: +# from django.contrib import admin +# admin.autodiscover() + +urlpatterns = patterns('', + # Examples: + # url(r'^$', 'clickcounter.views.home', name='home'), + # url(r'^clickcounter/', include('clickcounter.foo.urls')), + + # Uncomment the admin/doc line below to enable admin documentation: + # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + + # Uncomment the next line to enable the admin: + # url(r'^admin/', include(admin.site.urls)), +) diff --git a/examples/demoproject/settings.py b/examples/demoproject/settings.py index 7fca8e27..8951f826 100644 --- a/examples/demoproject/settings.py +++ b/examples/demoproject/settings.py @@ -18,14 +18,12 @@ MANAGERS = ADMINS -DATABASE_ENGINE = 'sqlite3' -DATABASE_NAME = 'testdb.sqlite' -DATABASE_USER = '' # Not used with sqlite3. -DATABASE_PASSWORD = '' # Not used with sqlite3. -DATABASE_HOST = '' # Set to empty string for localhost. - # Not used with sqlite3. -DATABASE_PORT = '' # Set to empty string for default. - # Not used with sqlite3. +DATABASES = {"default": {"NAME": "testdb.sqlite", + "ENGINE": "django.db.backends.sqlite3", + "USER": '', + "PASSWORD": '', + "HOST": '', + "PORT": ''}} INSTALLED_APPS = ( 'django.contrib.auth', diff --git a/requirements/test.txt b/requirements/test.txt index f4f08ee3..efa32e8e 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,4 +6,3 @@ mock django django-nose python-memcached - diff --git a/setup.py b/setup.py index 7a16fab7..e4c39e28 100755 --- a/setup.py +++ b/setup.py @@ -97,9 +97,22 @@ class QuickRunTests(RunTests): class CIRunTests(RunTests): - extra_args = ["--with-coverage3", "--with-xunit", - "--cover3-xml", "--xunit-file=nosetests.xml", - "--cover3-xml-file=coverage.xml"] + + @property + def extra_args(self): + toxinidir = os.environ.get("TOXINIDIR", "") + return [ + "--with-coverage3", + "--cover3-xml", + "--cover3-xml-file=%s" % ( + os.path.join(toxinidir, "coverage.xml"), ), + "--with-xunit", + "--xunit-file=%s" % ( + os.path.join(toxinidir, "nosetests.xml"), ), + "--cover3-html", + "--cover3-html-dir=%s" % ( + os.path.join(toxinidir, "cover"), ), + ] if os.path.exists("README.rst"): diff --git a/tests/settings.py b/tests/settings.py index 55afc6aa..6faad109 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -48,18 +48,12 @@ MANAGERS = ADMINS -DATABASE_ENGINE = 'sqlite3' -DATABASE_HOST = 'djcelery-test-db' -DATABASE_USER = '' -DATABASE_PASSWORD = '' -DATABASE_PORT = '' - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": "djcelery-test-db", - }, -} +DATABASES = {"default": {"NAME": "djcelery-test-db", + "ENGINE": "django.db.backends.sqlite3", + "USER": '', + "PASSWORD": '', + "PORT": ''}} + INSTALLED_APPS = ( 'django.contrib.auth', diff --git a/tox.ini b/tox.ini index 45722c93..aeb941a7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,32 +1,28 @@ [tox] -envlist = py24,py25,py26,py27 +envlist = py25,py26,py27 [testenv] distribute = True sitepackages = False -commands = nosetests -[testenv:py24] -basepython = python2.4 -commands = pip -E {envdir} install -r contrib/requirements/default.txt - pip -E {envdir} install -r contrib/requirements/test.txt - pip -E {envdir} install pysqlite - python setup.py citest - -[testenv:py25] -basepython = python2.5 -commands = pip -E {envdir} install -r contrib/requirements/default.txt - pip -E {envdir} install -r contrib/requirements/test.txt - python setup.py citest +[testenv:py27] +basepython = python2.7 +deps = -r{toxinidir}/requirements/default.txt + -r{toxinidir}/requirements/test.txt +commands = {toxinidir}/contrib/release/removepyc.sh {toxinidir} + env TOXINIDIR="{toxinidir}" python setup.py citest [testenv:py26] basepython = python2.6 -commands = pip -E {envdir} install -r contrib/requirements/default.txt - pip -E {envdir} install -r contrib/requirements/test.txt - python setup.py citest +deps = -r{toxinidir}/requirements/default.txt + -r{toxinidir}/requirements/test.txt +commands = {toxinidir}/contrib/release/removepyc.sh {toxinidir} + env TOXINIDIR="{toxinidir}" python setup.py citest + +[testenv:py25] +basepython = python2.5 +deps = -r{toxinidir}/requirements/default.txt + -r{toxinidir}/requirements/test.txt +commands = {toxinidir}/contrib/release/removepyc.sh {toxinidir} + env TOXINIDIR="{toxinidir}" python setup.py citest -[testenv:py27] -basepython = python2.7 -commands = pip -E {envdir} install -r contrib/requirements/default.txt - pip -E {envdir} install -r contrib/requirements/test.txt - python setup.py citest