Permalink
Browse files

fix bug 766256: Add rabbitmq & celery to vagrant

* Update celery & django-celery to latest in vendor

* Install & enable rabbitmq-server

* Configure rabbitmq-server with kuma user, vhost, and permissions

* Enable djcelery app in Kuma, add South migration for django-celery

* celery.decorators deprecated for celery.task

* Comment to use the real celery queue by setting
  CELERY_ALWAYS_EAGER=False

* Start up celeryd as part of ~/bin/go-tmux.sh

* Add mention of celeryd to /etc/motd

* Tweaks to /etc/hosts

* Don't leave ~/bin world-writable
  • Loading branch information...
1 parent 8b9878e commit a1d8f69598e8d50ba8404d45fd003963b6b1463c @lmorchard lmorchard committed Jun 20, 2012
@@ -8,7 +8,7 @@
from django.core import mail
from django.db.models import Q
-from celery.decorators import task
+from celery.task import task
from notifications.models import Watch, WatchFilter, EmailUser, multi_raw
from notifications.utils import merge, hash_to_unsigned
@@ -1,4 +1,4 @@
-from celery.decorators import task
+from celery.task import task
from notifications.models import Watch
@@ -5,7 +5,7 @@
from django.contrib.auth.models import User
from django.db import transaction
-from celery.decorators import task
+from celery.task import task
from questions import ANSWERS_PER_PAGE
@@ -5,7 +5,7 @@
from django.core.files.base import ContentFile
from PIL import Image
-from celery.decorators import task
+from celery.task import task
log = logging.getLogger('k.task')
View
@@ -9,7 +9,7 @@
from django.template import Context, loader
import celery.conf
-from celery.decorators import task
+from celery.task import task
from celery.messaging import establish_connection
from multidb.pinning import pin_this_thread, unpin_this_thread
from tower import ugettext as _
@@ -0,0 +1,182 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+
+ # Adding model 'IntervalSchedule'
+ db.create_table('djcelery_intervalschedule', (
+ ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('every', self.gf('django.db.models.fields.IntegerField')()),
+ ('period', self.gf('django.db.models.fields.CharField')(max_length=24)),
+ ))
+ db.send_create_signal('djcelery', ['IntervalSchedule'])
+
+ # Adding model 'CrontabSchedule'
+ db.create_table('djcelery_crontabschedule', (
+ ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('minute', self.gf('django.db.models.fields.CharField')(default='*', max_length=64)),
+ ('hour', self.gf('django.db.models.fields.CharField')(default='*', max_length=64)),
+ ('day_of_week', self.gf('django.db.models.fields.CharField')(default='*', max_length=64)),
+ ))
+ db.send_create_signal('djcelery', ['CrontabSchedule'])
+
+ # Adding model 'PeriodicTasks'
+ db.create_table('djcelery_periodictasks', (
+ ('ident', self.gf('django.db.models.fields.SmallIntegerField')(default=1, unique=True, primary_key=True)),
+ ('last_update', self.gf('django.db.models.fields.DateTimeField')()),
+ ))
+ db.send_create_signal('djcelery', ['PeriodicTasks'])
+
+ # Adding model 'PeriodicTask'
+ db.create_table('djcelery_periodictask', (
+ ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=200)),
+ ('task', self.gf('django.db.models.fields.CharField')(max_length=200)),
+ ('interval', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djcelery.IntervalSchedule'], null=True, blank=True)),
+ ('crontab', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djcelery.CrontabSchedule'], null=True, blank=True)),
+ ('args', self.gf('django.db.models.fields.TextField')(default='[]', blank=True)),
+ ('kwargs', self.gf('django.db.models.fields.TextField')(default='{}', blank=True)),
+ ('queue', self.gf('django.db.models.fields.CharField')(default=None, max_length=200, null=True, blank=True)),
+ ('exchange', self.gf('django.db.models.fields.CharField')(default=None, max_length=200, null=True, blank=True)),
+ ('routing_key', self.gf('django.db.models.fields.CharField')(default=None, max_length=200, null=True, blank=True)),
+ ('expires', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+ ('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
+ ('last_run_at', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+ ('total_run_count', self.gf('django.db.models.fields.PositiveIntegerField')(default=0)),
+ ('date_changed', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+ ))
+ db.send_create_signal('djcelery', ['PeriodicTask'])
+
+ # Adding model 'WorkerState'
+ db.create_table('djcelery_workerstate', (
+ ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('hostname', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255)),
+ ('last_heartbeat', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
+ ))
+ db.send_create_signal('djcelery', ['WorkerState'])
+
+ # Adding model 'TaskState'
+ db.create_table('djcelery_taskstate', (
+ ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('state', self.gf('django.db.models.fields.CharField')(max_length=64)),
+ ('task_id', self.gf('django.db.models.fields.CharField')(unique=True, max_length=36)),
+ ('name', self.gf('django.db.models.fields.CharField')(max_length=200, null=True, db_index=True)),
+ ('tstamp', self.gf('django.db.models.fields.DateTimeField')(db_index=True)),
+ ('args', self.gf('django.db.models.fields.TextField')(null=True)),
+ ('kwargs', self.gf('django.db.models.fields.TextField')(null=True)),
+ ('eta', self.gf('django.db.models.fields.DateTimeField')(null=True)),
+ ('expires', self.gf('django.db.models.fields.DateTimeField')(null=True)),
+ ('result', self.gf('django.db.models.fields.TextField')(null=True)),
+ ('traceback', self.gf('django.db.models.fields.TextField')(null=True)),
+ ('runtime', self.gf('django.db.models.fields.FloatField')(null=True)),
+ ('worker', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['djcelery.WorkerState'], null=True)),
+ ('hidden', self.gf('django.db.models.fields.BooleanField')(default=False)),
+ ))
+ db.send_create_signal('djcelery', ['TaskState'])
+
+
+ def backwards(self, orm):
+
+ # Deleting model 'IntervalSchedule'
+ db.delete_table('djcelery_intervalschedule')
+
+ # Deleting model 'CrontabSchedule'
+ db.delete_table('djcelery_crontabschedule')
+
+ # Deleting model 'PeriodicTasks'
+ db.delete_table('djcelery_periodictasks')
+
+ # Deleting model 'PeriodicTask'
+ db.delete_table('djcelery_periodictask')
+
+ # Deleting model 'WorkerState'
+ db.delete_table('djcelery_workerstate')
+
+ # Deleting model 'TaskState'
+ db.delete_table('djcelery_taskstate')
+
+
+ models = {
+ 'djcelery.crontabschedule': {
+ 'Meta': {'object_name': 'CrontabSchedule'},
+ 'day_of_week': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}),
+ 'hour': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'minute': ('django.db.models.fields.CharField', [], {'default': "'*'", 'max_length': '64'})
+ },
+ 'djcelery.intervalschedule': {
+ 'Meta': {'object_name': 'IntervalSchedule'},
+ 'every': ('django.db.models.fields.IntegerField', [], {}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'period': ('django.db.models.fields.CharField', [], {'max_length': '24'})
+ },
+ 'djcelery.periodictask': {
+ 'Meta': {'object_name': 'PeriodicTask'},
+ 'args': ('django.db.models.fields.TextField', [], {'default': "'[]'", 'blank': 'True'}),
+ 'crontab': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.CrontabSchedule']", 'null': 'True', 'blank': 'True'}),
+ 'date_changed': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'exchange': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}),
+ 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'interval': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.IntervalSchedule']", 'null': 'True', 'blank': 'True'}),
+ 'kwargs': ('django.db.models.fields.TextField', [], {'default': "'{}'", 'blank': 'True'}),
+ 'last_run_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}),
+ 'queue': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}),
+ 'routing_key': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}),
+ 'task': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+ 'total_run_count': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'})
+ },
+ 'djcelery.periodictasks': {
+ 'Meta': {'object_name': 'PeriodicTasks'},
+ 'ident': ('django.db.models.fields.SmallIntegerField', [], {'default': '1', 'unique': 'True', 'primary_key': 'True'}),
+ 'last_update': ('django.db.models.fields.DateTimeField', [], {})
+ },
+ 'djcelery.taskmeta': {
+ 'Meta': {'object_name': 'TaskMeta', 'db_table': "'celery_taskmeta'", 'managed': 'False'},
+ 'date_done': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'result': ('picklefield.fields.PickledObjectField', [], {'default': 'None', 'null': 'True'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'PENDING'", 'max_length': '50'}),
+ 'task_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'traceback': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
+ },
+ 'djcelery.tasksetmeta': {
+ 'Meta': {'object_name': 'TaskSetMeta', 'db_table': "'celery_tasksetmeta'", 'managed': 'False'},
+ 'date_done': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'result': ('picklefield.fields.PickledObjectField', [], {}),
+ 'taskset_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ },
+ 'djcelery.taskstate': {
+ 'Meta': {'ordering': "['-tstamp']", 'object_name': 'TaskState'},
+ 'args': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+ 'eta': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'hidden': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'kwargs': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'db_index': 'True'}),
+ 'result': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+ 'runtime': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'state': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+ 'task_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}),
+ 'traceback': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+ 'tstamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'worker': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['djcelery.WorkerState']", 'null': 'True'})
+ },
+ 'djcelery.workerstate': {
+ 'Meta': {'ordering': "['-last_heartbeat']", 'object_name': 'WorkerState'},
+ 'hostname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'last_heartbeat': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'})
+ }
+ }
+
+ complete_apps = ['djcelery']
No changes.
@@ -1,4 +1,4 @@
# Do not remove the following line, or various programs
# that require network functionality will fail.
-127.0.0.1 localhost kuma-master kuma-mdn developer-dev.mozilla.org
+127.0.0.1 localhost kuma developer-local.allizom.org developer-dev.mozilla.org
::1 localhost6.localdomain6 localhost6
@@ -12,6 +12,9 @@ To start the Django app:
To start the kumascript service:
node kumascript/run.js
+To start the celery queue workers:
+ ./manage.py celeryd
+
To run tests:
(cd / && /vagrant/manage.py test actioncounters contentflagging dekicompat demos devmo landing users wiki)
(cd /vagrant/kumascript && ./node_modules/.bin/nodeunit tests)
@@ -10,9 +10,11 @@ fi
tmux new-session -d -s $SESSION
+tmux split-window -t $SESSION:0 -v -p 50 './manage.py celeryd; /bin/bash'
+tmux swap-pane -t $SESSION:0 -U
tmux split-window -t $SESSION:0 -v -p 50 './manage.py runserver 0.0.0.0:8000; /bin/bash'
tmux swap-pane -t $SESSION:0 -U
tmux split-window -t $SESSION:0 -v -p 50 'node kumascript/run.js; /bin/bash'
-tmux select-pane -t $SESSION:0.2
+tmux select-pane -t $SESSION:0.3
tmux attach -t $SESSION
@@ -33,6 +33,9 @@
#EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
#EMAIL_FILE_PATH = '/home/vagrant/logs/kuma-email.log'
+# Uncomment to enable a real celery queue
+# CELERY_ALWAYS_EAGER = False
+
INSTALLED_APPS = INSTALLED_APPS + (
"django_extensions",
"debug_toolbar",
@@ -93,10 +93,10 @@
# owner => "root", group => "root", mode => 0440;
#}
- #file { "/etc/hosts":
- # source => "/vagrant/puppet/files/etc/hosts",
- # owner => "root", group => "root", mode => 0644;
- #}
+ file { "/etc/hosts":
+ source => "/vagrant/puppet/files/etc/hosts",
+ owner => "root", group => "root", mode => 0644;
+ }
# Disable SELinux... causing problems, and I don't understand it.
# TODO: see http://blog.endpoint.com/2010/02/selinux-httpd-modwsgi-26-rhel-centos-5.html
@@ -126,7 +126,7 @@
owner => "vagrant", group => "vagrant", mode => 0664;
"/home/vagrant/bin":
ensure => directory,
- owner => "vagrant", group => "vagrant", mode => 0777;
+ owner => "vagrant", group => "vagrant", mode => 0755;
"/home/vagrant/bin/go-tmux.sh":
source => "$PROJ_DIR/puppet/files/home/vagrant/bin/go-tmux.sh",
owner => "vagrant", group => "vagrant", mode => 0777;
@@ -0,0 +1,12 @@
+# Get rabbitmq up and running
+
+class rabbitmq {
+ package { ["rabbitmq-server"]:
+ ensure => present,
+ }
+ service { "rabbitmq-server":
+ ensure => running,
+ enable => true,
+ require => [ Package["rabbitmq-server"] ]
+ }
+}
@@ -64,18 +64,26 @@
}
-class sphinx_config {
- #exec {
- # "sphinx_reindex":
- # user => "vagrant",
- # cwd => "/vagrant",
- # command => "/home/vagrant/kuma-venv/bin/python ./manage.py reindex";
- # "sphinx_start":
- # user => "vagrant",
- # cwd => "/vagrant",
- # command => "/home/vagrant/kuma-venv/bin/python ./manage.py start_sphinx",
- # require => Exec['sphinx_reindex'];
- #}
+class rabbitmq_config {
+ exec {
+ 'rabbitmq-kuma-user':
+ require => [ Package['rabbitmq-server'], Service['rabbitmq-server'] ],
+ command => "/usr/sbin/rabbitmqctl add_user kuma kuma",
+ unless => "/usr/sbin/rabbitmqctl list_users 2>&1 | grep -q 'kuma'",
+ timeout => 300;
+ 'rabbitmq-kuma-vhost':
+ require => [ Package['rabbitmq-server'], Service['rabbitmq-server'],
+ Exec['rabbitmq-kuma-user'] ],
+ command => "/usr/sbin/rabbitmqctl add_vhost kuma",
+ unless => "/usr/sbin/rabbitmqctl list_vhosts 2>&1 | grep -q 'kuma'",
+ timeout => 300;
+ 'rabbitmq-kuma-permissions':
+ require => [ Package['rabbitmq-server'], Service['rabbitmq-server'],
+ Exec['rabbitmq-kuma-user'], Exec['rabbitmq-kuma-vhost'] ],
+ command => "/usr/sbin/rabbitmqctl set_permissions -p kuma kuma '.*' '.*' '.*'",
+ unless => "/usr/sbin/rabbitmqctl list_permissions -p kuma 2>&1 | grep -v 'vhost' | grep -q 'kuma'",
+ timeout => 300;
+ }
}
class kuma_config {
@@ -118,6 +126,6 @@
}
class site_config {
- include apache_config, mysql_config, sphinx_config, kuma_config
- Class[apache_config] -> Class[mysql_config] -> Class[sphinx_config] -> Class[kuma_config]
+ include apache_config, mysql_config, rabbitmq_config, kuma_config
+ Class[apache_config] -> Class[mysql_config] -> Class[rabbitmq_config] -> Class[kuma_config]
}
@@ -34,7 +34,7 @@
apache: stage => basics;
mysql: stage => basics;
memcache: stage => basics;
-# sphinx: stage => basics;
+ rabbitmq: stage => basics;
python: stage => langs;
php: stage => langs;
View
@@ -401,7 +401,7 @@ def lazy_language_deki_map():
# TODO: Reenable search when we switch to kuma wiki - or, at least waffle it.
'search',
#'forums',
- #'djcelery',
+ 'djcelery',
'notifications',
#'questions',
#'kadmin',
@@ -801,16 +801,18 @@ def read_only_mode(env):
BROKER_HOST = 'localhost'
BROKER_PORT = 5672
-BROKER_USER = 'kitsune'
-BROKER_PASSWORD = 'kitsune'
-BROKER_VHOST = 'kitsune'
+BROKER_USER = 'kuma'
+BROKER_PASSWORD = 'kuma'
+BROKER_VHOST = 'kuma'
CELERY_RESULT_BACKEND = 'amqp'
CELERY_IGNORE_RESULT = True
CELERY_ALWAYS_EAGER = True # For tests. Set to False for use.
CELERY_SEND_TASK_ERROR_EMAILS = True
CELERYD_LOG_LEVEL = logging.INFO
CELERYD_CONCURRENCY = 4
+CELERY_IMPORTS = ( 'wiki.tasks', )
+
# Wiki rebuild settings
WIKI_REBUILD_TOKEN = 'sumo:wiki:full-rebuild'
WIKI_REBUILD_ON_DEMAND = False
@@ -905,6 +907,7 @@ def read_only_mode(env):
'taggit': 'migrations.south.taggit',
# HACK: South treats "database" as the name of constance.backends.database
'database': 'migrations.south.constance',
+ 'djcelery': 'migrations.south.djcelery',
}
CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend'
Submodule vendor updated 842 files

0 comments on commit a1d8f69

Please sign in to comment.