diff --git a/.travis.yml b/.travis.yml index fe60c73..56636d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,18 @@ language: python -python: - - "2.7" - - "3.6" - - "pypy" - - "pypy3" -# command to install dependencies +cache: pip +matrix: + include: + - env: TOX_ENV=py35 + python: 3.5 + - env: TOX_ENV=py36 + python: 3.6 + - env: TOX_ENV=py37 + python: 3.7 + dist: bionic + sudo: true + install: - - "pip install -r requirements.txt" + - "pip install -r requirements.txt -r dev-requirements.txt tox tox-docker" - "pip install ." -# command to run tests -script: - - "python -m compileall ." + +script: tox -vv -e $TOX_ENV diff --git a/dev-requirements.txt b/dev-requirements.txt index 9646d48..1e6a38c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,6 +2,7 @@ astroid==2.3.3 bleach==3.1.0 certifi==2019.11.28 +celery[redis] chardet==3.0.4 docutils==0.16 idna==2.8 @@ -14,9 +15,9 @@ pkginfo==1.5.0.1 pygments==2.5.2 pylint==2.4.4 readme-renderer==24.0 +redis requests-toolbelt==0.9.1 requests==2.22.0 -six==1.14.0 tqdm==4.42.1 twine==3.1.1 typed-ast==1.4.1 ; implementation_name == 'cpython' and python_version < '3.8' diff --git a/djcelery_model/compat.py b/djcelery_model/compat.py new file mode 100644 index 0000000..61fa906 --- /dev/null +++ b/djcelery_model/compat.py @@ -0,0 +1,4 @@ +try: + from six import python_2_unicode_compatible +except ImportError: + from django.utils.encoding import python_2_unicode_compatible diff --git a/djcelery_model/models.py b/djcelery_model/models.py index 4447612..ec60a9f 100644 --- a/djcelery_model/models.py +++ b/djcelery_model/models.py @@ -9,7 +9,6 @@ from django.db.models import Q from django.db.models.query import QuerySet from django.contrib.contenttypes.models import ContentType -from django.utils.encoding import python_2_unicode_compatible try: # Django >= 1.7 @@ -21,6 +20,8 @@ from celery.utils import uuid from celery import signals +from .compat import python_2_unicode_compatible + class ModelTaskMetaState(object): PENDING = 0 STARTED = 1 diff --git a/djcelery_model/tests/__init__.py b/djcelery_model/tests/__init__.py new file mode 100644 index 0000000..070e835 --- /dev/null +++ b/djcelery_model/tests/__init__.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import, unicode_literals + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ('celery_app',) diff --git a/djcelery_model/tests/celery.py b/djcelery_model/tests/celery.py new file mode 100644 index 0000000..4b23784 --- /dev/null +++ b/djcelery_model/tests/celery.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import, unicode_literals +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djcelery_model.tests.settings') + +app = Celery('project') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + +from celery.contrib.testing.tasks import ping diff --git a/djcelery_model/tests/settings.py b/djcelery_model/tests/settings.py new file mode 100644 index 0000000..385560b --- /dev/null +++ b/djcelery_model/tests/settings.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 +from __future__ import unicode_literals, absolute_import +import os + +DEBUG = True +USE_TZ = True + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz' + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sites', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + + 'djcelery_model', + 'djcelery_model.tests.testapp', +] + +SITE_ID = 1 + +MIDDLEWARE = [ + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', +] + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + os.path.join(os.path.dirname(__file__), 'templates'), + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +ROOT_DIR = os.path.dirname(__file__) +STATIC_ROOT = os.path.join(ROOT_DIR, 'test-static') +MEDIA_ROOT = os.path.join(ROOT_DIR, 'test-media') +MEDIA_URL = '/media/' +STATIC_URL = '/static/' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + } +} + +REDIS_PORT = os.getenv('REDIS_6379_TCP_PORT', 'port-missing-from-env') + +CELERY_BROKER_URL = 'redis://localhost:%s/0' % REDIS_PORT +CELERY_RESULT_BACKEND = 'redis://localhost:%s/2' % REDIS_PORT +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_SEND_SENT_EVENT = True +CELERY_SEND_EVENTS = True diff --git a/djcelery_model/tests/testapp/__init__.py b/djcelery_model/tests/testapp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/djcelery_model/tests/testapp/migrations/0001_initial.py b/djcelery_model/tests/testapp/migrations/0001_initial.py new file mode 100644 index 0000000..99cfc06 --- /dev/null +++ b/djcelery_model/tests/testapp/migrations/0001_initial.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.7 on 2019-11-10 23:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='JPEGFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('file', models.FileField(upload_to='')), + ('etag', models.CharField(blank=True, max_length=255)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/djcelery_model/tests/testapp/migrations/__init__.py b/djcelery_model/tests/testapp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/djcelery_model/tests/testapp/models.py b/djcelery_model/tests/testapp/models.py new file mode 100644 index 0000000..0913036 --- /dev/null +++ b/djcelery_model/tests/testapp/models.py @@ -0,0 +1,6 @@ +from django.db import models +from djcelery_model.models import TaskMixin + +class JPEGFile(TaskMixin, models.Model): + file = models.FileField() + etag = models.CharField(max_length=255, blank=True) diff --git a/djcelery_model/tests/testapp/static/testapp/flower.jpg b/djcelery_model/tests/testapp/static/testapp/flower.jpg new file mode 100644 index 0000000..933719d Binary files /dev/null and b/djcelery_model/tests/testapp/static/testapp/flower.jpg differ diff --git a/djcelery_model/tests/testapp/tasks.py b/djcelery_model/tests/testapp/tasks.py new file mode 100644 index 0000000..b1dee61 --- /dev/null +++ b/djcelery_model/tests/testapp/tasks.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, unicode_literals +from hashlib import sha1 +from time import sleep +from celery import shared_task + +from .models import JPEGFile + + +@shared_task +def calculate_etag(pk): + jpeg = JPEGFile.objects.get(pk=pk) + jpeg.etag = sha1(jpeg.file.read()).hexdigest() + sleep(5) + jpeg.save() diff --git a/djcelery_model/tests/testapp/tests.py b/djcelery_model/tests/testapp/tests.py new file mode 100644 index 0000000..e2565b2 --- /dev/null +++ b/djcelery_model/tests/testapp/tests.py @@ -0,0 +1,64 @@ +from time import sleep +from celery.contrib.testing.tasks import ping +from celery.contrib.testing.worker import start_worker +from django.contrib.staticfiles import finders +from django.core.files import File +from django.test import TestCase, TransactionTestCase + +from djcelery_model.models import TaskMixin +from djcelery_model.tests import celery_app + +from .models import JPEGFile +from .tasks import calculate_etag + + +class CeleryTestCase(TransactionTestCase): + def setUp(self): + self.worker_context = start_worker(celery_app, perform_ping_check=False) + self.worker = self.worker_context.__enter__() + self.worker.ensure_started() + + def tearDown(self): + self.worker_context.__exit__(None, None, None) + + +class TestAppIntegrationTests(TestCase): + def test_model_is_taskmixin(self): + self.assertIsInstance(JPEGFile(), TaskMixin) + + +class TestAppCeleryTests(CeleryTestCase): + def test_worker(self): + result = ping.delay() + pong = result.get(timeout=10) + self.assertEqual(pong, 'pong') + + def test_state_properties(self): + jpeg = JPEGFile.objects.create( + file=File(open(finders.find('testapp/flower.jpg'), 'rb'), name='flower.jpg') + ) + + self.assertFalse(jpeg.has_running_task) + self.assertFalse(jpeg.has_ready_task) + + result = jpeg.apply_async(calculate_etag, [jpeg.pk]) + self.assertTrue(jpeg.has_running_task) + self.assertFalse(jpeg.has_ready_task) + self.assertFalse(result.ready()) + + result.get(timeout=10) + self.assertTrue(result.ready()) + + # not the greatest way to wait for async stuff to happen, but we need + # the signals to complete before testing for side effects + sleep(3) + + self.assertEqual(jpeg.etag, '') + self.assertFalse(jpeg.has_running_task) + self.assertTrue(jpeg.has_ready_task) + + jpeg.refresh_from_db() + + self.assertEqual(jpeg.etag, '80b098e6cd95b9901fa29799d48731433dfaeab0') + self.assertFalse(jpeg.has_running_task) + self.assertTrue(jpeg.has_ready_task) diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..2a4c384 --- /dev/null +++ b/manage.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djcelery_model.tests.settings") + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/requirements.txt b/requirements.txt index b546fa8..79e177e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ django>=1.11 celery>=4.2 +six>=1.9 diff --git a/runtests.py b/runtests.py new file mode 100644 index 0000000..44650fa --- /dev/null +++ b/runtests.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import os +import shutil +import sys +import warnings + +from django.core.management import execute_from_command_line + +os.environ['DJANGO_SETTINGS_MODULE'] = 'djcelery_model.tests.settings' + + +def runtests(): + # Don't ignore DeprecationWarnings + only_djcelery_model = r'^djcelery_model(\.|$)' + warnings.filterwarnings('default', category=DeprecationWarning, module=only_djcelery_model) + warnings.filterwarnings('default', category=PendingDeprecationWarning, module=only_djcelery_model) + + args = sys.argv[1:] + argv = sys.argv[:1] + ['test'] + args + try: + execute_from_command_line(argv) + finally: + from djcelery_model.tests.settings import STATIC_ROOT, MEDIA_ROOT + shutil.rmtree(STATIC_ROOT, ignore_errors=True) + shutil.rmtree(MEDIA_ROOT, ignore_errors=True) + + +if __name__ == '__main__': + runtests() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..e572008 --- /dev/null +++ b/tox.ini @@ -0,0 +1,32 @@ +[tox] +envlist = + {py35,py36,py37} + + +[testenv] +setenv = + PYTHONPATH = {toxinidir}:{toxinidir} + +commands = + python runtests.py --noinput {posargs} + +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/dev-requirements.txt + +basepython = + py37: python3.7 + py36: python3.6 + py35: python3.5 + +docker = + redis:5.0 + + + +[docker:redis:5.0] +healthcheck_cmd = redis-cli ping | grep -q PONG +healthcheck_interval = 3 +healthcheck_timeout = 3 +healthcheck_retries = 30 +healthcheck_start_period = 5 diff --git a/vagrant/.gitignore b/vagrant/.gitignore new file mode 100644 index 0000000..a977916 --- /dev/null +++ b/vagrant/.gitignore @@ -0,0 +1 @@ +.vagrant/ diff --git a/vagrant/README b/vagrant/README new file mode 100644 index 0000000..81dbeed --- /dev/null +++ b/vagrant/README @@ -0,0 +1,23 @@ +CREATES A VAGRANT ENVIRONMENT TO FACILITATE LOCAL TESTING + +======== +USAGE: +======== +To run tests: + cd to this directory and then issue the following commands: + vagrant up + vagrant ssh + bash /vagrant/vagrant/runtox.sh + + to run with verbose output: + bash /vagrant/vagrant/runtox.sh -vv -- -v 2 + + to run a specific test: + bash /vagrant/vagrant/runtox.sh -vv -- -v 2 path.to.test + + +To clean after tests run: + rm -R /tmp/vagrant + +To make migrations (after running tests to create tox environment): + /tmp/vagrant/.tox/py35/bin/python /vagrant/manage.py makemigrations diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile new file mode 100644 index 0000000..2ddff7a --- /dev/null +++ b/vagrant/Vagrantfile @@ -0,0 +1,11 @@ +Vagrant.configure("2") do |config| + config.vm.box = "hashicorp/bionic64" + + config.vm.provider "virtualbox" do |v| + v.name = "django-celery-model" + end + + config.vm.provision :shell, path: "provision.sh" + config.vm.synced_folder ".", "/vagrant", disabled: true + config.vm.synced_folder "../", "/vagrant" +end diff --git a/vagrant/provision.sh b/vagrant/provision.sh new file mode 100644 index 0000000..caba974 --- /dev/null +++ b/vagrant/provision.sh @@ -0,0 +1,17 @@ +set -o errexit +set -o pipefail +set -o nounset +shopt -s failglob +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +add-apt-repository -y ppa:deadsnakes/ppa +apt-get update +apt-get install -y build-essential libssl-dev python3.5 python3.5-dev python3.6 python3.6-dev python3.7 python3.7-dev + +curl -sSL https://get.docker.com/ | sh +adduser vagrant docker + +curl https://bootstrap.pypa.io/get-pip.py | python3 +pip3 install tox tox-docker diff --git a/vagrant/runtox.sh b/vagrant/runtox.sh new file mode 100644 index 0000000..b520da9 --- /dev/null +++ b/vagrant/runtox.sh @@ -0,0 +1,5 @@ +rsync --recursive --exclude="*/node_modules/*" --exclude="*/.git/*" /vagrant /tmp + +cd /tmp/vagrant + +tox "$@"