Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ sudo: false
python:
- '2.7'
- '3.4'
- '3.6'
env:
- DJANGO_VERSION=1.7
- DJANGO_VERSION=1.8
- DJANGO_VERSION=1.9
- DJANGO_VERSION=1.10
- DJANGO_VERSION=1.11
install:
- pip install -r test-requirements.txt
- pip install -U django==$DJANGO_VERSION
Expand Down
4 changes: 1 addition & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ Simple databased-backed job queue. Jobs are defined in your settings, and are pr

Asynchronous tasks are run via a *job queue*. This system is designed to support multi-step job workflows.

**NOTE**: This module uses differing implementations of UUIDField on Django 1.7 and 1.8 - a Python 3 shimmed django-uuidfield version on 1.7, and the built-in implementation on Django 1.8 and above. The simplest way to upgrade it is to drop the existing
`django_dbq_job` table, delete the migration from `django_migrations`, and then
re-run `manage.py migrate`.
Tested against Django 1.8, 1.9, 1.10, 1.11

## Getting Started

Expand Down
2 changes: 1 addition & 1 deletion django_dbq/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '1.0.1'
__version__ = '1.2.0'
23 changes: 16 additions & 7 deletions django_dbq/management/commands/create_job.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django_dbq.models import Job
from optparse import make_option
import json
import logging

Expand All @@ -14,12 +13,22 @@ class Command(BaseCommand):
help = "Create a job"
args = '<job_name>'

option_list = BaseCommand.option_list + (
make_option('--workspace',
help='JSON-formatted initial command workspace'),
make_option('--queue_name',
help='A specific queue to add this job to'),
)
def add_arguments(self, parser):
parser.add_argument('args', nargs='+')
parser.add_argument(
'--workspace',
action='store_true',
dest='workspace',
default=None,
help="JSON-formatted initial commandworkspace."
)
parser.add_argument(
'--queue_name',
action='store_true',
dest='queue_name',
default=None,
help="A specific queue to add this job to"
)

def handle(self, *args, **options):
if len(args) != 1:
Expand Down
17 changes: 9 additions & 8 deletions django_dbq/management/commands/worker.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from django.db import transaction
from django.core.management.base import BaseCommand, CommandError
from django.utils.module_loading import import_by_path
from django.utils.module_loading import import_string
from django_dbq.models import Job
from optparse import make_option
from simplesignals.process import WorkerProcessBase
from time import sleep
import logging
Expand All @@ -27,7 +26,7 @@ def process_job(queue_name):
job.save()

try:
task_function = import_by_path(job.next_task)
task_function = import_string(job.next_task)
task_function(job)
job.update_next_task()
if not job.next_task:
Expand All @@ -41,7 +40,7 @@ def process_job(queue_name):
failure_hook_name = job.get_failure_hook_name()
if failure_hook_name:
logger.info("Running failure hook %s for job id=%s", failure_hook_name, job.pk)
failure_hook_function = import_by_path(failure_hook_name)
failure_hook_function = import_string(failure_hook_name)
failure_hook_function(job, exception)
else:
logger.info("No failure hook for job id=%s", job.pk)
Expand Down Expand Up @@ -72,12 +71,14 @@ class Command(BaseCommand):

help = "Run a queue worker process"

option_list = BaseCommand.option_list + (
make_option('--dry-run',
def add_arguments(self, parser):
parser.add_argument('queue_name', nargs='?', default='default', type=str)
parser.add_argument(
'--dry-run',
action='store_true',
dest='dry_run',
default=False,
help="Don't actually start the worker. Used for testing."),
help="Don't actually start the worker. Used for testing."
)

def handle(self, *args, **options):
Expand All @@ -87,7 +88,7 @@ def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Please supply a single queue job name")

queue_name = args[0]
queue_name = options['queue_name']

self.stdout.write("Starting job worker for queue \"%s\"" % queue_name)

Expand Down
5 changes: 2 additions & 3 deletions django_dbq/models.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from django.db import models
from django.utils.module_loading import import_by_path
from django.utils.module_loading import import_string
from django_dbq.tasks import get_next_task_name, get_failure_hook_name, get_creation_hook_name
from jsonfield import JSONField
from model_utils import Choices
Expand Down Expand Up @@ -103,6 +103,5 @@ def run_creation_hook(self):
creation_hook_name = self.get_creation_hook_name()
if creation_hook_name:
logger.info("Running creation hook %s for new job", creation_hook_name)
creation_hook_function = import_by_path(creation_hook_name)
creation_hook_function = import_string(creation_hook_name)
creation_hook_function(self)

3 changes: 1 addition & 2 deletions django_dbq/tests.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from datetime import datetime, timedelta
from django.core.management import call_command, CommandError
from django.core.urlresolvers import reverse
from django.test import TestCase
from django.test.utils import override_settings
from django_dbq.management.commands.worker import process_job
Expand Down Expand Up @@ -73,7 +72,7 @@ def test_worker_no_args(self):

def test_worker_with_queue_name(self):
stdout = StringIO()
call_command('worker', 'test_queue', dry_run=True, stdout=stdout)
call_command('worker', queue_name='test_queue', dry_run=True, stdout=stdout)
output = stdout.getvalue()
self.assertTrue('test_queue' in output)

Expand Down
2 changes: 1 addition & 1 deletion test-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
-r requirements.txt
pymysql==0.6.7
pymysql==0.7.11