Skip to content
This repository
Browse code

Merge branch 'celery'

  • Loading branch information...
commit ae6f685eb61aff8fe162698788433bb5485b7260 2 parents 68164cb + d7455f8
Paul Watts authored April 09, 2012
2  .gitignore
@@ -8,7 +8,9 @@ media/resized_image
8 8
 media/front
9 9
 media/staff
10 10
 media/members
  11
+media/arp/import.log
11 12
 .project
12 13
 .pydevproject
13 14
 .settings
  15
+celerybeat-schedule.db
14 16
 
8  README.markdown
Source Rendered
@@ -17,8 +17,10 @@ Set up PostgreSQL, create a blank database and grant all permissions to whatever
17 17
 Copy local_settings.dist to local_settings.py and edit it to reflect your local settings. 
18 18
 
19 19
 Run Django's syncdb and then South's migrate commands.  
  20
+(Currently creating a superuser before running migrate is broken; when prompted to create one, choose no.
  21
+After running the <code>migrate</code> command, run <code>./manage.py createsuperuser</code>.)
20 22
 
21  
-    ./manage.py syncdb # when prompted, create an admin account
  23
+    ./manage.py syncdb 
22 24
     ./manage.py migrate
23 25
 
24 26
 Now run the tests to make certain that everthing is installed:
@@ -45,7 +47,7 @@ And visit your installation of Nadine at http://127.0.0.1:8000/
45 47
 
46 48
 In order to repeatedly execute tasks like checking and sending email, run this command:
47 49
 
48  
-    ./manage.py scheduler
  50
+    ./manage.py celeryd -B
49 51
 
50 52
 You will need to run that command as a long lived process.  On linux and other unices, use something like the nohup command.
51 53
 
@@ -74,7 +76,7 @@ In the interest of shipping more quickly, we have made certain assumptions about
74 76
 - the reply-to address for mail from a list is the original sender, not the entire list
75 77
 - attachments are neither saved nor sent to the list, but a removal note is appended to the message
76 78
 - incoming messages are parsed for a single text message and a single html message (not multiple MIME messages)
77  
-- you can set the frequency of mail fetching in the EmailTask in your local_settings.py
  79
+- you can set the frequency of mail fetching by changing the value in CELERYBEAT_SCHEDULE in your settings.py or local_settings.py
78 80
 - loops and bounces are silently dropped
79 81
 - any email sent to a list which is not in a subscriber's user or membership record is moderated
80 82
 - the sender of a message receives a copy of the message like any other subscriber
0  front/management/__init__.py
No changes.
0  front/management/commands/__init__.py
No changes.
22  front/management/commands/scheduler.py
... ...
@@ -1,22 +0,0 @@
1  
-import os
2  
-import time
3  
-import urllib
4  
-import sys
5  
-import datetime
6  
-
7  
-from django.conf import settings
8  
-from django.core.management.base import BaseCommand, CommandError
9  
-
10  
-from front.scheduler import Scheduler
11  
-
12  
-class Command(BaseCommand):
13  
-   help = "Runs the process which schedules Tasks from settings.SCHEDULED_TASKS."
14  
-   args = ""
15  
-   requires_model_validation = True
16  
-
17  
-   def handle(self, *labels, **options):
18  
-      scheduler = Scheduler()
19  
-      for task in settings.SCHEDULED_TASKS: scheduler.add_task(task)
20  
-      scheduler.start_all_tasks()
21  
-
22  
-# Copyright 2011 Office Nomads LLC, Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
20  front/models.py
... ...
@@ -1,27 +1,11 @@
1  
-import os
2  
-import os.path
3  
-from datetime import datetime, timedelta, date
4  
-import random
5  
-import time
6  
-import re
7  
-import unicodedata
  1
+from datetime import datetime
8 2
 import traceback
9  
-import logging
10  
-import pprint
11 3
 
12  
-from django.template.loader import render_to_string
13  
-from django.utils.html import strip_tags
14 4
 from django.db import models
15  
-from django.db.models import signals
16 5
 from django.conf import settings
17  
-from django.contrib.auth.models import User
18  
-from django.contrib.sites.models import Site
19  
-from django.dispatch import dispatcher
20  
-from django.core.mail import send_mail
21  
-from django.utils.encoding import force_unicode
22  
-from django.db.models import Q
23 6
 from django.core.mail import send_mail
24 7
 
  8
+
25 9
 class EmailEntryManager(models.Manager):
26 10
    def unsent_entries(self): return self.filter(sent__isnull=True).order_by('created')
27 11
 
61  front/scheduler.py
... ...
@@ -1,61 +0,0 @@
1  
-#!/usr/bin/python
2  
-"""
3  
-A task scheduling script which schedules tasks defined in settings.SCHEDULED_TASKS.
4  
-"""
5  
-import cmd
6  
-import time
7  
-import logging
8  
-import readline
9  
-import datetime
10  
-import threading
11  
-import traceback
12  
-
13  
-ONE_HOUR_SECONDS = 60 * 60
14  
-ONE_DAY_SECONDS = ONE_HOUR_SECONDS * 24
15  
-
16  
-class Task(threading.Thread):
17  
-   def __init__(self, action, loopdelay, initdelay):
18  
-      """The action is a function which will be called in a new thread every loopdelay microseconds, starting after initdelay microseconds"""
19  
-      self._action = action
20  
-      self._loopdelay = loopdelay
21  
-      self._initdelay = initdelay
22  
-      self._running = 1
23  
-      self.last_alert_datetime = None
24  
-      threading.Thread.__init__(self)
25  
-
26  
-   def run(self):
27  
-      """There's no need to override this.  Pass your action in as a function to the __init__."""
28  
-      if self._initdelay: time.sleep(self._initdelay)
29  
-      self._runtime = time.time()
30  
-      while self._running:
31  
-         start = time.time()
32  
-         self._action()
33  
-         self._runtime += self._loopdelay
34  
-         time.sleep(max(0, self._runtime - start))
35  
-
36  
-   def stop(self): self._running = 0
37  
-
38  
-class Scheduler:
39  
-   """The class which manages the starting and stopping of tasks."""
40  
-   def __init__(self):
41  
-      self._tasks = []
42  
-
43  
-   def __repr__(self): return '\n'.join(['%s' % task for task in self._tasks])
44  
-
45  
-   def add_task(self, task): self._tasks.append(task)
46  
-
47  
-   def start_all_tasks(self):
48  
-      print 'Starting scheduler'
49  
-      for task in self._tasks:
50  
-         print 'Starting task', task
51  
-         task.start()
52  
-      print 'All tasks started'
53  
-
54  
-   def stop_all_tasks(self):
55  
-      for task in self._tasks:
56  
-         print 'Stopping task', task
57  
-         task.stop()
58  
-         task.join()
59  
-      print 'Stopped'
60  
-
61  
-# Copyright 2010 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
6  front/tasks.py
... ...
@@ -1,6 +0,0 @@
1  
-from datetime import datetime, timedelta
2  
-import traceback
3  
-
4  
-from front.scheduler import Task
5  
-
6  
-# Copyright 2010 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
49  interlink/admin.py
... ...
@@ -1,32 +1,49 @@
5  interlink/forms.py
... ...
@@ -1,5 +1,4 @@
@@ -7,11 +6,11 @@
45  interlink/mail.py
... ...
@@ -1,17 +1,22 @@
@@ -30,11 +35,11 @@ def add_test_incoming(mailing_list, origin_address, subject, body, sent_time=Non
@@ -47,29 +52,35 @@ def fetch_mail(self):
@@ -77,7 +88,7 @@ def fetch_mail(self):
@@ -95,7 +106,7 @@ def fetch_mail(self):
@@ -114,12 +125,12 @@ def find_bodies(self, message):
241  interlink/models.py
... ...
@@ -1,31 +1,18 @@
@@ -35,14 +22,14 @@ def user_by_email(email):
@@ -54,27 +41,25 @@ def awaiting_moderation(user):
@@ -85,25 +70,26 @@ class MailingList(models.Model):
@@ -113,39 +99,21 @@ def moderator_addresses(self):
@@ -180,6 +148,26 @@ def create_outgoing(self):
@@ -193,9 +181,27 @@ def __unicode__(self): return '%s: %s' % (self.origin_address, self.subject)
@@ -214,55 +220,54 @@ class OutgoingMail(models.Model):
29  interlink/tasks.py
... ...
@@ -1,23 +1,14 @@
54  interlink/tests/list_tests.py
... ...
@@ -1,21 +1,18 @@
@@ -24,9 +21,9 @@ def setUp(self):
@@ -41,7 +38,7 @@ def test_subscription_form(self):
@@ -49,9 +46,9 @@ def test_moderator_controlled(self):
@@ -61,7 +58,7 @@ def test_moderator_controlled(self):
@@ -70,53 +67,54 @@ def test_moderator_controlled(self):
@@ -131,7 +129,7 @@ def test_incoming_processing(self):
@@ -141,7 +139,7 @@ def test_incoming_processing(self):
@@ -154,7 +152,7 @@ def test_incoming_processing(self):
@@ -164,7 +162,7 @@ def test_recipients(self):
@@ -174,9 +172,9 @@ def test_mail_checking(self):
5  local_settings.dist
@@ -32,11 +32,6 @@ EMAIL_USE_TLS = True
32 32
 EMAIL_PORT = 587
33 33
 EMAIL_SUBJECT_PREFIX = "[COWORKING] " # or None if you want no subject prefix
34 34
 
35  
-from interlink.tasks import EmailTask
36  
-from staff.tasks import MailingListTask, BillingTask
37  
-# You can set the frequency of any Task by passing loopdelay=<number of seconds>
38  
-SCHEDULED_TASKS = (EmailTask(), MailingListTask(), BillingTask())
39  
-
40 35
 import datetime
41 36
 BILLING_START_DATE = datetime.date(2009, 11, 17)
42 37
 NEW_MEMBER_DEPOSIT = 500
2  requirements.txt
@@ -5,3 +5,5 @@ pil
5 5
 feedparser
6 6
 django-taggit
7 7
 django-taggit-templatetags
  8
+celery==2.5.1
  9
+django-celery==2.5.1
40  settings.py
@@ -2,6 +2,8 @@
2 2
 import os
3 3
 import sys
4 4
 
  5
+from datetime import timedelta
  6
+
5 7
 PROJECT_ROOT = os.path.realpath(os.path.dirname(__file__))
6 8
 TEMPLATE_DIRS = ( PROJECT_ROOT + '/templates/', )
7 9
 MEDIA_ROOT = PROJECT_ROOT + '/media/'
@@ -100,6 +102,7 @@
100 102
 	'django.contrib.humanize',
101 103
 	'django.contrib.staticfiles',
102 104
 	'taggit',
  105
+	'djcelery',
103 106
 	'taggit_templatetags',
104 107
 	'south',
105 108
 	'front',
@@ -110,6 +113,43 @@
110 113
 	'tablet',
111 114
 )
112 115
 
  116
+#
  117
+# Celery initialization
  118
+#
  119
+try:
  120
+   import djcelery
  121
+   djcelery.setup_loader()
  122
+except ImportError:
  123
+   pass
  124
+
  125
+BROKER_URL = "amqp://guest:guest@localhost:5672//"
  126
+
  127
+#
  128
+# Celery beat schedules
  129
+#
  130
+CELERYBEAT_SCHEDULE = {
  131
+	"email-task": {
  132
+		"task": "interlink.tasks.email_task",
  133
+		"schedule": timedelta(seconds=2),
  134
+	},
  135
+	#"billing-task": {
  136
+   #  "task": "staff.tasks.billing_task",
  137
+	#	"schedule": timedelta(hours=1)
  138
+	#},
  139
+	"unsubscribe-dropouts": {
  140
+		"task": "staff.tasks.unsubscribe_recent_dropouts_task",
  141
+		"schedule": timedelta(hours=1)
  142
+	},
  143
+}
  144
+CELERY_DISABLE_RATE_LIMITS = True
  145
+CELERY_RESULT_BACKEND = "amqp"
  146
+
  147
+# When this is True, celery tasks will be run synchronously.
  148
+# This is nice when running unit tests or in development.
  149
+# In production set this to False in your local_settings.py
  150
+CELERY_ALWAYS_EAGER = False
  151
+
  152
+
113 153
 from local_settings import *
114 154
 
115 155
 # Copyright 2009, 2010 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
44  staff/tasks.py
... ...
@@ -1,37 +1,27 @@
1  
-import traceback
2  
-from datetime import datetime, date, timedelta
  1
+from celery.task import task
3 2
 
4  
-from django.core.exceptions import ObjectDoesNotExist
  3
+from datetime import datetime, timedelta
5 4
 
6  
-from front.scheduler import Task, ONE_DAY_SECONDS, ONE_HOUR_SECONDS
7 5
 
8  
-class BillingTask(Task):
  6
+@task(ignore_result=True)
  7
+def billing_task():
9 8
 	"""
10 9
 	A recurring task which calculates billing.
11 10
 	The task runs once an hour, but will only run billing once every 24 hours.
12 11
 	"""
13  
-	def __init__(self, loopdelay=ONE_HOUR_SECONDS, initdelay=1):
14  
-		Task.__init__(self, self.perform_task, loopdelay, initdelay)
15  
-		self.name = "BillingRunTask"
16  
-
17  
-	def perform_task(self):
18  
-		from staff import billing
19  
-		from staff.models import BillingLog
20  
-		try:
21  
-			latest_billing_log = BillingLog.objects.latest()
22  
-			if latest_billing_log.started > datetime.now() - timedelta(hours=24): return
23  
-		except ObjectDoesNotExist:
24  
-			pass
25  
-		billing.run_billing()		
26  
-
27  
-class MailingListTask(Task):
  12
+	import billing
  13
+	from models import BillingLog
  14
+
  15
+	last_day = datetime.now() - timedelta(hours=24)
  16
+	if not BillingLog.objects.filter(started__gt=last_day).exists():
  17
+		billing.run_billing()
  18
+
  19
+
  20
+@task(ignore_result=True)
  21
+def unsubscribe_recent_dropouts_task():
28 22
 	"""A recurring task which checks for members who need to be unsubscribed from mailing lists"""
29  
-	def __init__(self, loopdelay=3600, initdelay=5):
30  
-		Task.__init__(self, self.perform_task, loopdelay, initdelay)
31  
-		self.name = "MailingListTask"
  23
+	from models import Member
  24
+	Member.objects.unsubscribe_recent_dropouts()
32 25
 
33  
-	def perform_task(self):
34  
-		from staff.models import Member
35  
-		Member.objects.unsubscribe_recent_dropouts()      
36 26
 
37  
-# Copyright 2011 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  27
+# Copyright 2012 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

0 notes on commit ae6f685

Please sign in to comment.
Something went wrong with that request. Please try again.