update requirements, migrate to celery 4, prepare 1.0.0 release, fixes …

…#32

.gitignore

@@ -7,3 +7,4 @@ dist/
 build/
 *.sw*
 .coverage
+celerybeat-schedule.db

.travis.yml

@@ -6,10 +6,10 @@ python:
   - 2.7
   - 3.5
   - 3.6
+  - 3.7
 
 env:
-  - DJANGO_VERSION=1.10.5
-  - DJANGO_VERSION=1.11.9
+  - DJANGO_VERSION=1.11.20
   - DJANGO_VERSION=2.1
 
 matrix:

CHANGELOG.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+1.0.0 (2019-02-13)
+------------------
+
+- BREAKING CHANGE! switch to celery 4, you now have to add the scheduler task to your CELERYBEAT_SCHEDULE, details in README.md [Jens Nistler]
+- BREAKING CHANGE! removed DJANGO_DATAWATCH_CELERY_QUEUE_NAME setting, use task routing instead, see http://docs.celeryproject.org/en/latest/userguide/routing.html [Jens Nistler]
+- Update dependencies [Jens Nistler]
+- It's time for a 1.0.0 release since datawatch is used internally at RegioHelden for over two years now [Jens Nistler]
+
+
 0.3.1 (2018-08-07)
 ------------------
 

Dockerfile

@@ -24,12 +24,14 @@ USER app
 
 ADD requirements.txt /app/
 
+ADD requirements-docker.txt /app/
+
 ENV PATH /home/app/venv/bin:${PATH}
 
 RUN pyvenv ~/venv && \
     pip install --upgrade pip && \
-    pip install wheel pip-tools && \
-    pip-sync
+    pip install wheel && \
+    pip install -r requirements-docker.txt
 
 ADD . /app/
 

README.md

@@ -21,6 +21,9 @@ $ pip install django-datawatch
 
 Add `django_datawatch` to your `INSTALLED_APPS`
 
+Add `django_datawatch.tasks.django_datawatch_scheduler` to the `CELERYBEAT_SCHEDULE` of your app.
+This task should be executed every minute e.g. `crontab(minute='*/1')`, see example app.
+
 ## Write a custom check
 
 Create `checks.py` inside your module.
@@ -128,7 +131,6 @@ $ ./manage.py datawatch_delete_ghost_results
 
 ```python
 DJANGO_DATAWATCH_BACKEND = 'django_datawatch.backends.synchronous'
-DJANGO_DATAWATCH_CELERY_QUEUE_NAME = 'django_datawatch'
 DJANGO_DATAWATCH_RUN_SIGNALS = True
 ```
 
@@ -138,18 +140,18 @@ You can chose the backend to run the tasks. Supported are 'django_datawatch.back
 
 Default: 'django_datawatch.backends.synchronous'
 
-### DJANGO_DATAWATCH_CELERY_QUEUE_NAME
-
-You can customize the celery queue name for async tasks (applies only if celery backend chosen).
-
-Default: 'django_datawatch'
-
 ### DJANGO_DATAWATCH_RUN_SIGNALS
 
 Use this setting to disable running post_save updates during unittests if required.
 
 Default: True
 
+### celery task queue
+
+Datawatch supported setting a specific queue in release < 0.4.0
+
+With the switch to celery 4, you should use task routing to define the queue for your tasks, see http://docs.celeryproject.org/en/latest/userguide/routing.html
+
 # CONTRIBUTE
 
 ## Dev environment
@@ -180,8 +182,8 @@ docker-compose up -d
 
 Then setup the example app environment.
 ```bash
-docker-compose exec django ./manage.py migrate
-docker-compose exec django ./manage.py loaddata example
+docker-compose run --rm django migrate
+docker-compose run --rm django loaddata example
 ```
 The installed superuser is "example" with password "datawatch".
 
@@ -191,13 +193,18 @@ Login on the admin interface and open http://datawatch.rh-dev.eu:8000/ afterward
 You'll be prompted with an empty dashboard. That's because we didn't run any checks yet.
 Let's enqueue an update.
 ```bash
-docker-compose exec django ./manage.py datawatch_run_checks --force
+docker-compose run --rm django datawatch_run_checks --force
 ```
 
 The checks for the example app are run synchronously and should be updated immediately.
 If you decide to switch to the celery backend, you should now start a celery worker to process the checks.
 ```bash
-docker-compose exec django celery worker -A example -l DEBUG -Q django_datawatch
+docker-compose run --rm --entrypoint=celery django worker -A example -l DEBUG
+```
+
+To execute the celery beat scheduler which runs the datawatch scheduler every minute, just run:
+```bash
+docker-compose run --rm --entrypoint=celery django beat --scheduler django_celery_beat.schedulers:DatabaseScheduler -A example
 ```
 
 You will see some failed check now after you refreshed the dashboard view.

django_datawatch/backends/base.py

@@ -2,11 +2,11 @@
 
 
 class BaseBackend(object):
-    def enqueue(self, slug, async=True):
+    def enqueue(self, slug, run_async=True):
         raise NotImplementedError('enqueue not implemented')
 
-    def refresh(self, slug, async=True):
+    def refresh(self, slug, run_async=True):
         raise NotImplementedError('refresh not implemented')
 
-    def run(self, slug, identifier, async=True):
+    def run(self, slug, identifier, run_async=True):
         raise NotImplementedError('run not implemented')

django_datawatch/backends/celery.py

@@ -13,32 +13,23 @@ class Backend(BaseBackend):
     """
     A wrapper backend to execute tasks asynchronously in celery
     """
-    def enqueue(self, slug, async=True):
-        kwargs = dict(kwargs=dict(slug=slug),
-                      queue=getattr(settings,
-                                    'DJANGO_DATAWATCH_CELERY_QUEUE_NAME',
-                                    defaults['CELERY_QUEUE_NAME']))
-        if async:
+    def enqueue(self, slug, run_async=True):
+        kwargs = dict(kwargs=dict(slug=slug))
+        if run_async:
             django_datawatch_enqueue.apply_async(**kwargs)
         else:
             django_datawatch_enqueue.apply(**kwargs)
 
-    def refresh(self, slug, async=True):
-        kwargs = dict(kwargs=dict(slug=slug),
-                      queue=getattr(settings,
-                                    'DJANGO_DATAWATCH_CELERY_QUEUE_NAME',
-                                    defaults['CELERY_QUEUE_NAME']))
-        if async:
+    def refresh(self, slug, run_async=True):
+        kwargs = dict(kwargs=dict(slug=slug))
+        if run_async:
             django_datawatch_refresh.apply_async(**kwargs)
         else:
             django_datawatch_refresh.apply(**kwargs)
 
-    def run(self, slug, identifier, async=True):
-        kwargs = dict(kwargs=dict(slug=slug, identifier=identifier),
-                      queue=getattr(settings,
-                                    'DJANGO_DATAWATCH_CELERY_QUEUE_NAME',
-                                    defaults['CELERY_QUEUE_NAME']))
-        if async:
+    def run(self, slug, identifier, run_async=True):
+        kwargs = dict(kwargs=dict(slug=slug, identifier=identifier))
+        if run_async:
             django_datawatch_run.apply_async(**kwargs)
         else:
             django_datawatch_run.apply(**kwargs)

django_datawatch/backends/synchronous.py

@@ -11,7 +11,7 @@
 
 
 class Backend(BaseBackend):
-    def enqueue(self, slug, async=True):
+    def enqueue(self, slug, run_async=True):
         check = self._get_check_instance(slug)
         if not check:
             return
@@ -23,11 +23,11 @@ def enqueue(self, slug, async=True):
         except NotImplementedError as e:
             logger.error(e)
 
-    def refresh(self, slug, async=True):
+    def refresh(self, slug, run_async=True):
         for result in Result.objects.filter(slug=slug):
             datawatch.get_backend().run(result.slug, result.identifier)
 
-    def run(self, slug, identifier, async=True):
+    def run(self, slug, identifier, run_async=True):
         check = self._get_check_instance(slug)
         if not check:
             return

django_datawatch/datawatch.py

@@ -114,7 +114,7 @@ def update_related(self, sender, instance):
                 return
 
             backend.run(slug=check.slug, identifier=check.get_identifier(payload),
-                        async=True)
+                        run_async=True)
 
 
 datawatch = DatawatchHandler()

django_datawatch/management/commands/datawatch_refresh_results.py

@@ -27,4 +27,4 @@ def handle(self, slug, *args, **options):
 
     def refresh(self, slug):
         backend = datawatch.get_backend()
-        backend.refresh(slug=slug, async=True)
+        backend.refresh(slug=slug, run_async=True)

django_datawatch/tasks.py

@@ -32,10 +32,6 @@ def django_datawatch_run(slug, identifier, *args, **kwargs):
     synchronous.Backend().run(slug, identifier)
 
 
-class DatawatchScheduler(PeriodicTask):
-    run_every = crontab(minute='*/1')
-    queue = getattr(settings, 'DJANGO_DATAWATCH_CELERY_QUEUE_NAME',
-                    defaults['CELERY_QUEUE_NAME'])
-
-    def run(self, *args, **kwargs):
-        Scheduler().run_checks(force=False)
+@shared_task
+def django_datawatch_scheduler(*args, **kwargs):
+    Scheduler().run_checks(force=False)

django_datawatch/views.py

@@ -142,7 +142,7 @@ def form_valid(self, form):
 
         datawatch.get_backend().run(slug=check.slug,
                                     identifier=check.get_identifier(self.object),
-                                    async=False)
+                                    run_async=False)
         return super(ResultConfigView, self).form_valid(form)
 
 
@@ -162,7 +162,7 @@ def get(self, request, *args, **kwargs):
         check = self.object.get_check_instance()
         datawatch.get_backend().run(slug=check.slug,
                                     identifier=self.object.identifier,
-                                    async=False)
+                                    run_async=False)
         messages.add_message(request, messages.INFO, _('Result has been refreshed'))
         return response
 

docker-compose.yaml

@@ -30,17 +30,21 @@ services:
       - db_data:/var/lib/postgresql/data
     restart: unless-stopped
     networks:
-      - default
+      default:
+        aliases:
+          - datawatch-db
 
   rabbitmq:
-    image: rabbitmq:3.6-management
+    image: rabbitmq:3.7-management
     ports:
       - '15672:15672'  # admin UI; credentials: guest/guest
     volumes:
       - /var/lib/rabbitmq
     restart: unless-stopped
     networks:
-      - default
+      default:
+        aliases:
+          - datawatch-rabbitmq
 
 volumes:
   db_data: {}

example/celery.py

@@ -5,6 +5,7 @@
 
 from django.apps import apps
 from celery import Celery
+from celery.schedules import crontab
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'example.settings')
@@ -15,3 +16,10 @@
 # pickle the object when using Windows.
 app.config_from_object('django.conf:settings')
 app.autodiscover_tasks(lambda: [n.name for n in apps.get_app_configs()])
+
+app.conf.CELERYBEAT_SCHEDULE.update({
+    'datawatch_scheduler': {
+        'task': 'django_datawatch.tasks.django_datawatch_scheduler',
+        'schedule': crontab(minute='*/1'),
+    },
+})

example/settings.py

@@ -25,7 +25,9 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = []
+ALLOWED_HOSTS = [
+    '.datawatch.rh-dev.eu',
+]
 
 
 # Application definition
@@ -38,6 +40,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'bootstrap3',
+    'django_celery_beat',
     'django_datawatch.apps.DjangoDatawatchConfig',
     'example.apps.ExampleConfig',
 ]
@@ -94,7 +97,7 @@
             'NAME': 'app',
             'USER': 'app',
             'PASSWORD': 'app',
-            'HOST': 'db',
+            'HOST': 'datawatch-db',
             'PORT': '5432',
         },
     }
@@ -169,12 +172,12 @@
 
 STATIC_URL = '/static/'
 
-BROKER_URL = 'amqp://rabbitmq'
+BROKER_URL = 'amqp://datawatch-rabbitmq'
 
 LOGIN_URL = '/admin/'
 
 SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
 
 BOOTSTRAP3 = {'horizontal_label_class': 'col-md-2', 'horizontal_field_class': 'col-md-10', 'success_css_class': ''}
 
-DJANGO_DATAWATCH_BACKEND = 'django_datawatch.backends.synchronous'
+DJANGO_DATAWATCH_BACKEND = 'django_datawatch.backends.celery'

requirements-docker.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+django_celery_beat==1.4.0

requirements-test.txt

@@ -1,9 +1,9 @@
-celery==3.1.23
-django-bootstrap3==7.1.0
-django-extensions==1.7.4
-django-model-utils==2.6
-flake8==3.5.0
-freezegun==0.3.7
+celery==4.2.1
+django-bootstrap3==11.0.0
+django-extensions==2.1.5
+django-model-utils==3.1.2
+flake8==3.7.5
+freezegun==0.3.11
 mock==2.0.0
-psycopg2==2.6.2
-python-dateutil==2.5.3
+psycopg2==2.7.7
+python-dateutil==2.8.0

requirements.txt

@@ -1,10 +1,10 @@
-celery==3.1.23
-Django==1.10.2
-django-bootstrap3==7.1.0
-django-extensions==1.7.4
-django-model-utils==2.6
-freezegun==0.3.7
-ipython==5.1.0
+celery==4.2.1
+Django==1.11.20
+django-bootstrap3==11.0.0
+django-extensions==2.1.5
+django-model-utils==3.1.2
+freezegun==0.3.11
+ipython==7.2.0
 mock==2.0.0
-psycopg2==2.6.2
-python-dateutil==2.5.3
+psycopg2==2.7.7
+python-dateutil==2.8.0