Permalink
Browse files

Merge branch 'celery'

  • Loading branch information...
2 parents 68164cb + d7455f8 commit ae6f685eb61aff8fe162698788433bb5485b7260 @paulcwatts paulcwatts committed Apr 9, 2012
View
2 .gitignore
@@ -8,7 +8,9 @@ media/resized_image
media/front
media/staff
media/members
+media/arp/import.log
.project
.pydevproject
.settings
+celerybeat-schedule.db
View
8 README.markdown
@@ -17,8 +17,10 @@ Set up PostgreSQL, create a blank database and grant all permissions to whatever
Copy local_settings.dist to local_settings.py and edit it to reflect your local settings.
Run Django's syncdb and then South's migrate commands.
+(Currently creating a superuser before running migrate is broken; when prompted to create one, choose no.
+After running the <code>migrate</code> command, run <code>./manage.py createsuperuser</code>.)
- ./manage.py syncdb # when prompted, create an admin account
+ ./manage.py syncdb
./manage.py migrate
Now run the tests to make certain that everthing is installed:
@@ -45,7 +47,7 @@ And visit your installation of Nadine at http://127.0.0.1:8000/
In order to repeatedly execute tasks like checking and sending email, run this command:
- ./manage.py scheduler
+ ./manage.py celeryd -B
You will need to run that command as a long lived process. On linux and other unices, use something like the nohup command.
@@ -74,7 +76,7 @@ In the interest of shipping more quickly, we have made certain assumptions about
- the reply-to address for mail from a list is the original sender, not the entire list
- attachments are neither saved nor sent to the list, but a removal note is appended to the message
- incoming messages are parsed for a single text message and a single html message (not multiple MIME messages)
-- you can set the frequency of mail fetching in the EmailTask in your local_settings.py
+- you can set the frequency of mail fetching by changing the value in CELERYBEAT_SCHEDULE in your settings.py or local_settings.py
- loops and bounces are silently dropped
- any email sent to a list which is not in a subscriber's user or membership record is moderated
- the sender of a message receives a copy of the message like any other subscriber
View
0 front/management/__init__.py
No changes.
View
0 front/management/commands/__init__.py
No changes.
View
22 front/management/commands/scheduler.py
@@ -1,22 +0,0 @@
-import os
-import time
-import urllib
-import sys
-import datetime
-
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-
-from front.scheduler import Scheduler
-
-class Command(BaseCommand):
- help = "Runs the process which schedules Tasks from settings.SCHEDULED_TASKS."
- args = ""
- requires_model_validation = True
-
- def handle(self, *labels, **options):
- scheduler = Scheduler()
- for task in settings.SCHEDULED_TASKS: scheduler.add_task(task)
- scheduler.start_all_tasks()
-
-# Copyright 2011 Office Nomads LLC, Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
View
20 front/models.py
@@ -1,27 +1,11 @@
-import os
-import os.path
-from datetime import datetime, timedelta, date
-import random
-import time
-import re
-import unicodedata
+from datetime import datetime
import traceback
-import logging
-import pprint
-from django.template.loader import render_to_string
-from django.utils.html import strip_tags
from django.db import models
-from django.db.models import signals
from django.conf import settings
-from django.contrib.auth.models import User
-from django.contrib.sites.models import Site
-from django.dispatch import dispatcher
-from django.core.mail import send_mail
-from django.utils.encoding import force_unicode
-from django.db.models import Q
from django.core.mail import send_mail
+
class EmailEntryManager(models.Manager):
def unsent_entries(self): return self.filter(sent__isnull=True).order_by('created')
View
61 front/scheduler.py
@@ -1,61 +0,0 @@
-#!/usr/bin/python
-"""
-A task scheduling script which schedules tasks defined in settings.SCHEDULED_TASKS.
-"""
-import cmd
-import time
-import logging
-import readline
-import datetime
-import threading
-import traceback
-
-ONE_HOUR_SECONDS = 60 * 60
-ONE_DAY_SECONDS = ONE_HOUR_SECONDS * 24
-
-class Task(threading.Thread):
- def __init__(self, action, loopdelay, initdelay):
- """The action is a function which will be called in a new thread every loopdelay microseconds, starting after initdelay microseconds"""
- self._action = action
- self._loopdelay = loopdelay
- self._initdelay = initdelay
- self._running = 1
- self.last_alert_datetime = None
- threading.Thread.__init__(self)
-
- def run(self):
- """There's no need to override this. Pass your action in as a function to the __init__."""
- if self._initdelay: time.sleep(self._initdelay)
- self._runtime = time.time()
- while self._running:
- start = time.time()
- self._action()
- self._runtime += self._loopdelay
- time.sleep(max(0, self._runtime - start))
-
- def stop(self): self._running = 0
-
-class Scheduler:
- """The class which manages the starting and stopping of tasks."""
- def __init__(self):
- self._tasks = []
-
- def __repr__(self): return '\n'.join(['%s' % task for task in self._tasks])
-
- def add_task(self, task): self._tasks.append(task)
-
- def start_all_tasks(self):
- print 'Starting scheduler'
- for task in self._tasks:
- print 'Starting task', task
- task.start()
- print 'All tasks started'
-
- def stop_all_tasks(self):
- for task in self._tasks:
- print 'Stopping task', task
- task.stop()
- task.join()
- print 'Stopped'
-
-# Copyright 2010 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
View
6 front/tasks.py
@@ -1,6 +0,0 @@
-from datetime import datetime, timedelta
-import traceback
-
-from front.scheduler import Task
-
-# Copyright 2010 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
View
49 interlink/admin.py
@@ -1,32 +1,49 @@
from django.contrib import admin
-from django import forms
-from django.forms.util import ErrorList
-from django.contrib.auth.models import User
-from django.contrib.auth.admin import UserAdmin
from models import MailingList, IncomingMail, OutgoingMail
-class MailingListAdmin(admin.ModelAdmin):
+class MailBase(admin.ModelAdmin):
+ def _action(self, request, queryset, f, singular, plural):
+ count = 0
+ for u in queryset.iterator():
+ f(u)
+ count = count + 1
+ if count == 1:
+ self.message_user(request, singular)
+ else:
+ self.message_user(request, plural % count)
+
+class MailingListAdmin(MailBase):
raw_id_fields = ['subscribers']
+ actions = ['fetch_mail']
+ def fetch_mail(self, request, queryset):
+ self._action(request, queryset,
+ lambda u: u.fetch_mail(),
+ "Mail fetched from 1 list",
+ "Mail fetched from %s lists")
+ fetch_mail.short_description = "Fetch mail"
admin.site.register(MailingList, MailingListAdmin)
-class IncomingMailAdmin(admin.ModelAdmin):
+class IncomingMailAdmin(MailBase):
list_display = ('sent_time', 'origin_address', 'subject', 'state')
+ actions = ['process_mail']
+ def process_mail(self, request, queryset):
+ self._action(request, queryset,
+ lambda u: u.process(),
+ "1 incoming mail processed",
+ "%s incoming mails processed")
+ process_mail.short_description = "Process mail"
admin.site.register(IncomingMail, IncomingMailAdmin)
-class OutgoingMailAdmin(admin.ModelAdmin):
+class OutgoingMailAdmin(MailBase):
list_display = ('id', 'original_mail', 'subject', 'sent')
-
+
actions = ['send_mail']
def send_mail(self, request, queryset):
- mail_queued = 0
- for u in queryset.iterator():
- u.send()
- mail_queued = mail_queued + 1
- if mail_queued == 1:
- self.message_user(request, "1 mail queued to send")
- else:
- self.message_user(request, "%d mails queued to send" % mail_queued)
+ self._action(request, queryset,
+ lambda u: u.send(),
+ "1 mail sent",
+ "%s mails sent")
send_mail.short_description = "Send mail"
admin.site.register(OutgoingMail, OutgoingMailAdmin)
View
5 interlink/forms.py
@@ -1,17 +1,16 @@
from django import forms
-from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from interlink.models import MailingList, OutgoingMail
class MailingListSubscriptionForm(forms.Form):
subscribe = forms.CharField(required=False, widget=forms.HiddenInput)
mailing_list_id = forms.IntegerField(required=True, widget=forms.HiddenInput)
-
+
def save(self, user):
list = MailingList.objects.get(pk=self.cleaned_data['mailing_list_id'])
if list.moderator_controlled: return False
-
+
body = 'So says http://%s ' % Site.objects.get_current().domain
if self.cleaned_data['subscribe'] == 'true' and (user.get_profile().is_monthly() or user.is_staff):
list.subscribers.add(user)
View
45 interlink/mail.py
@@ -1,17 +1,22 @@
-import time
-import poplib, email
-from datetime import datetime, date, timedelta
+import poplib
+import email
+import logging
+from datetime import datetime
from django.conf import settings
from django.utils.html import strip_tags
from interlink.models import IncomingMail
from django.contrib.sites.models import Site
+
+logger = logging.getLogger(__name__)
+
class MailChecker(object):
"""An abstract class for fetching mail from something like a pop or IMAP server"""
- def __init__(self, mailing_list):
+ def __init__(self, mailing_list, _logger=None):
self.mailing_list = mailing_list
-
+ self.logger = _logger or logger
+
def fetch_mail(self):
"""
Talk to the remote service, fetch the mail, create IncomingMail records
@@ -30,11 +35,11 @@ def add_test_incoming(mailing_list, origin_address, subject, body, sent_time=Non
class TestMailChecker(MailChecker):
"""A queue based mock object used in test suites."""
-
- def __init__(self, mailing_list):
- super(TestMailChecker, self).__init__(mailing_list)
+
+ def __init__(self, mailing_list, logger=None):
+ super(TestMailChecker, self).__init__(mailing_list, logger)
if not TEST_INCOMING_MAIL.has_key(mailing_list): TEST_INCOMING_MAIL[mailing_list] = []
-
+
def fetch_mail(self):
"""
Pops mail from the TEST_INCOMING_MAIL queue and creates IncomingMail records for them
@@ -47,37 +52,43 @@ def fetch_mail(self):
return results
class PopMailChecker(MailChecker):
-
+
def fetch_mail(self):
"""Pops mail from the pop server and writes them as IncomingMail"""
+ self.logger.debug("Checking mail from %s:%d" %
+ (self.mailing_list.pop_host, self.mailing_list.pop_port))
pop_client = poplib.POP3_SSL(self.mailing_list.pop_host, self.mailing_list.pop_port)
response = pop_client.user(self.mailing_list.username)
if not response.startswith('+OK'): raise Exception('Username not accepted: %s' % response)
response = pop_client.pass_(self.mailing_list.password)
if not response.startswith('+OK Logged in'): raise Exception('Password not accepted: %s' % response)
stats = pop_client.stat()
- if stats[0] == 0: return []
+ if stats[0] == 0:
+ self.logger.debug("No mail")
+ return []
results = []
+ self.logger.debug("Processing %d mails" % stats[0])
for i in range(stats[0]):
- response, mail, size = pop_client.retr(i+1)
+ response, mail, _size = pop_client.retr(i+1)
parser = email.FeedParser.FeedParser()
parser.feed('\n'.join(mail))
message = parser.close()
+
# Delete and ignore auto responses
if message['Auto-Submitted'] and message['Auto-Submitted'] != 'no':
pop_client.dele(i+1)
continue
-
+
# Delete and ignore messages sent from any list to avoid loops
if message['List-ID']:
pop_client.dele(i+1)
continue
#TODO Delete and ignore soft bounces
- name, origin_address = email.utils.parseaddr(message['From'])
+ _name, origin_address = email.utils.parseaddr(message['From'])
time_struct = email.utils.parsedate(message['Date'])
if time_struct:
sent_time = datetime(*time_struct[:-2])
@@ -95,7 +106,7 @@ def fetch_mail(self):
results.append(IncomingMail.objects.create(mailing_list=self.mailing_list, origin_address=origin_address, subject=message['Subject'], body=body, html_body=html_body, sent_time=sent_time, original_message=message))
pop_client.dele(i+1)
-
+
pop_client.quit()
def find_bodies(self, message):
@@ -114,12 +125,12 @@ def find_bodies(self, message):
elif bod.has_key('Content-Disposition') and bod['Content-Disposition'].startswith('attachment; filename="'):
file_names.append(bod['Content-Disposition'][len('attachment; filename="'):-1])
return (body, html_body, file_names)
-
+
if settings.IS_TEST:
DEFAULT_MAIL_CHECKER = TestMailChecker
else:
DEFAULT_MAIL_CHECKER = PopMailChecker
-
+
# Copyright 2011 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
View
241 interlink/models.py
@@ -1,31 +1,18 @@
-import os
-import re
-import time
-import email
-import random
-import logging
-import smtplib
-import traceback
-import unicodedata
-from email.mime.text import MIMEText
-from email.mime.multipart import MIMEMultipart
-from datetime import datetime, timedelta, date
+from datetime import datetime, timedelta
+from collections import defaultdict
from django.db import models
from django.db.models import Q
-from django.conf import settings
-from django.db.models import signals
-from django.dispatch import dispatcher
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse
+from django.core.mail import get_connection, EmailMultiAlternatives
from django.db.models.signals import post_save
-from django.utils.encoding import force_unicode
from django.template.loader import render_to_string
-from django.core.mail import send_mail, send_mass_mail
from staff.models import Member, Membership
+
def user_by_email(email):
users = User.objects.filter(email__iexact=email)
if len(users) > 0: return users[0]
@@ -35,14 +22,14 @@ def user_by_email(email):
User.objects.find_by_email = user_by_email
def membership_save_callback(sender, **kwargs):
- """When a membership is created, add the user to any opt-out mailing lists"""
- membership = kwargs['instance']
- created = kwargs['created']
- if not created: return
- # If the member is just switching from one membership to another, don't change subscriptions
- if Membership.objects.filter(member=membership.member, end_date=membership.start_date-timedelta(days=1)).count() != 0: return
- mailing_lists = MailingList.objects.filter(is_opt_out=True)
- for ml in mailing_lists: ml.subscribers.add(membership.member.user)
+ """When a membership is created, add the user to any opt-out mailing lists"""
+ membership = kwargs['instance']
+ created = kwargs['created']
+ if not created: return
+ # If the member is just switching from one membership to another, don't change subscriptions
+ if Membership.objects.filter(member=membership.member, end_date=membership.start_date-timedelta(days=1)).count() != 0: return
+ mailing_lists = MailingList.objects.filter(is_opt_out=True)
+ for ml in mailing_lists: ml.subscribers.add(membership.member.user)
post_save.connect(membership_save_callback, sender=Membership)
@@ -54,27 +41,25 @@ def awaiting_moderation(user):
class MailingListManager(models.Manager):
def unsubscribe_from_all(self, user):
for ml in self.all():
- if user in ml.subscribers.all(): ml.subscribers.remove(user)
-
- def fetch_all_mail(self):
+ if user in ml.subscribers.all():
+ ml.subscribers.remove(user)
+
+ def fetch_all_mail(self, logger=None):
"""Fetches mail for all mailing lists and returns an array of mailing_lists which reported failures"""
- failures = []
- for mailing_list in self.all():
- if not mailing_list.fetch_mail():
- failures.append(mailing_list)
- return failures
+ for ml in self.all():
+ ml.fetch_mail(logger)
class MailingList(models.Model):
"""Represents both the user facing information about a mailing list and how to fetch the mail"""
name = models.CharField(max_length=1024)
description = models.TextField(blank=True)
subject_prefix = models.CharField(max_length=1024, blank=True)
-
+
is_opt_out = models.BooleanField(default=False, help_text='True if new users should be automatically enrolled')
moderator_controlled = models.BooleanField(default=False, help_text='True if only the moderators can send mail to the list and can unsubscribe users.')
email_address = models.EmailField()
-
+
username = models.CharField(max_length=1024)
password = models.CharField(max_length=1024)
@@ -85,25 +70,26 @@ class MailingList(models.Model):
subscribers = models.ManyToManyField(User, blank=True, related_name='subscribed_mailing_lists')
moderators = models.ManyToManyField(User, blank=True, related_name='moderated_mailing_lists', help_text='Users who will be sent moderation emails', limit_choices_to={'is_staff': True})
-
+
objects = MailingListManager()
-
- def fetch_mail(self):
- """Fetches mailing and returns True if successful and False if it failed"""
+
+ def fetch_mail(self, logger=None):
+ """Fetches incoming mails from the mailing list."""
from interlink.mail import DEFAULT_MAIL_CHECKER
- checker = DEFAULT_MAIL_CHECKER(self)
- try:
- checker.fetch_mail()
- return True
- except:
- traceback.print_exc()
- return False
+ checker = DEFAULT_MAIL_CHECKER(self, logger)
+ checker.fetch_mail()
+
+ def get_smtp_connection(self):
+ return get_connection(host=self.smtp_host,
+ port=self.smtp_port,
+ username=self.username,
+ password=self.password)
@property
def list_id(self):
"""Used for List-ID mail headers"""
return '%s <%s-%s>' % (self.name, Site.objects.get_current().domain, self.id)
-
+
@property
def moderator_addresses(self):
"""Returns a tuple of email address strings, one for each moderator address"""
@@ -113,39 +99,21 @@ def moderator_addresses(self):
def subscriber_addresses(self):
"""Returns a tuple of email address strings, one for each subscribed address"""
return tuple([sub.email for sub in self.subscribers.all()])
-
+
def __unicode__(self): return '%s' % self.name
@models.permalink
def get_absolute_url(self): return ('interlink.views.list', (), { 'id':self.id })
def user_mailing_list_memberships(user):
- """Returns an array of tuples of <MailingList, is_subscriber> for a User"""
- return [(ml, user in ml.subscribers.all()) for ml in MailingList.objects.all()]
+ """Returns an array of tuples of <MailingList, is_subscriber> for a User"""
+ return [(ml, user in ml.subscribers.all()) for ml in MailingList.objects.all()]
User.mailing_list_memberships = user_mailing_list_memberships
class IncomingMailManager(models.Manager):
def process_incoming(self):
for incoming in self.filter(state='raw'):
- incoming.owner = User.objects.find_by_email(incoming.origin_address)
-
- if incoming.mailing_list.moderator_controlled:
- if incoming.owner in incoming.mailing_list.moderators.all():
- incoming.create_outgoing()
- else:
- incoming.state = 'reject'
- incoming.save()
- continue
-
- if incoming.owner == None or not incoming.owner in incoming.mailing_list.subscribers.all():
- subject = 'Moderation Request: %s: %s' % (incoming.mailing_list.name, incoming.subject)
- body = render_to_string('interlink/email/moderation_required.txt', { 'incoming_mail': incoming })
- OutgoingMail.objects.create(mailing_list=incoming.mailing_list, moderators_only=True, original_mail=incoming, subject=subject, body=body)
- incoming.state = 'moderate'
- incoming.save()
- continue
- else:
- incoming.create_outgoing()
+ incoming.process()
class IncomingMail(models.Model):
"""An email as popped for a mailing list"""
@@ -180,6 +148,26 @@ def create_outgoing(self):
self.save()
return outgoing
+ def process(self):
+ self.owner = User.objects.find_by_email(self.origin_address)
+
+ if self.mailing_list.moderator_controlled:
+ if self.owner in self.mailing_list.moderators.all():
+ self.create_outgoing()
+ else:
+ self.state = 'reject'
+ self.save()
+
+ elif self.owner == None or not self.owner in self.mailing_list.subscribers.all():
+ subject = 'Moderation Request: %s: %s' % (self.mailing_list.name, self.subject)
+ body = render_to_string('interlink/email/moderation_required.txt', { 'incoming_mail': self })
+ OutgoingMail.objects.create(mailing_list=self.mailing_list, moderators_only=True, original_mail=self, subject=subject, body=body)
+ self.state = 'moderate'
+ self.save()
+
+ else:
+ self.create_outgoing()
+
@property
def approve_url(self): return 'http://%s%s' % (Site.objects.get_current().domain, reverse('interlink.views.moderator_approve', kwargs={'id':self.id}, current_app='interlink'))
@@ -193,9 +181,27 @@ def __unicode__(self): return '%s: %s' % (self.origin_address, self.subject)
class OutgoingMailManager(models.Manager):
def send_outgoing(self):
- for mail in self.filter(sent=None):
- if mail.last_attempt and mail.last_attempt > datetime.now() - timedelta(minutes=10): continue
- mail.send()
+ # First, get all the mails we want to send
+ to_send = (self.select_related('mailing_list')
+ .filter(sent__isnull=True)
+ .filter(Q(last_attempt__isnull=True) |
+ Q(last_attempt__lt=datetime.now() - timedelta(minutes=10))))
+ # This dict is indexed by the mailing list
+ # and contains a list of each mail that should be sent using that smtp info
+ d = defaultdict(list)
+ for m in to_send:
+ d[m.mailing_list].append(m)
+
+ # Once we have this, we can go through each key value pair,
+ # make a connection to the server, and send them all.
+ for ml, mails in d.iteritems():
+ try:
+ conn = ml.get_smtp_connection()
+ conn.open()
+ for m in mails:
+ m.send(conn)
+ finally:
+ conn.close()
class OutgoingMail(models.Model):
"""Emails which are consumed by the interlink.tasks.EmailTask"""
@@ -214,55 +220,54 @@ class OutgoingMail(models.Model):
objects = OutgoingMailManager()
- def send(self):
- if self.sent: return False
+ def send(self, connection=None):
+ if self.sent:
+ return False
+
self.last_attempt = datetime.now()
self.attempts = self.attempts + 1
self.save()
- try:
- msg = MIMEMultipart('alternative')
- if self.body: msg.attach(MIMEText(self.body, 'plain', 'utf-8'))
- if self.html_body: msg.attach(MIMEText(self.html_body, 'html', 'utf-8'))
-
- msg['To'] = self.mailing_list.email_address
- if self.original_mail and self.original_mail.owner:
- msg['From'] = '"%s" <%s>' % (self.original_mail.owner.get_full_name(), self.mailing_list.email_address)
- else:
- msg['From'] = self.mailing_list.email_address
- msg['Subject'] = self.subject
- msg['Date'] = email.utils.formatdate()
- if self.original_mail:
- msg['Reply-To'] = self.original_mail.origin_address
- else:
- msg['Reply-To'] = self.mailing_list.email_address
- msg['List-ID'] = self.mailing_list.list_id
- msg['X-CAN-SPAM-1'] = 'This message may be a solicitation or advertisement within the specific meaning of the CAN-SPAM Act of 2003.'
-
- if self.moderators_only:
- recipient_addresses = self.mailing_list.moderator_addresses
- else:
- recipient_addresses = self.mailing_list.subscriber_addresses
-
- if not settings.IS_TEST and not msg['To'] == '':
- try:
- smtp_server = smtplib.SMTP(self.mailing_list.smtp_host, self.mailing_list.smtp_port)
- smtp_server.login(self.mailing_list.username, self.mailing_list.password)
- smtp_server.sendmail(msg['From'], recipient_addresses + (self.mailing_list.email_address,), msg.as_string())
- smtp_server.quit()
- except:
- traceback.print_exc()
- return False
-
- if self.original_mail and self.original_mail.state != 'moderate':
- self.original_mail.state = 'sent'
- self.original_mail.save()
-
- self.sent = datetime.now()
- self.save()
- return True
- except:
- traceback.print_exc()
- return False
+
+ if self.moderators_only:
+ to = self.mailing_list.moderator_addresses
+ else:
+ to = self.mailing_list.subscriber_addresses
+
+ if self.original_mail and self.original_mail.owner:
+ from_email = '"%s" <%s>' % (self.original_mail.owner.get_full_name(), self.mailing_list.email_address)
+ else:
+ from_email = self.mailing_list.email_address
+
+ headers = {
+ #'Date': email.utils.formatdate(), # Done by default in Django
+ 'List-ID': self.mailing_list.list_id,
+ 'X-CAN-SPAM-1': 'This message may be a solicitation or advertisement within the specific meaning of the CAN-SPAM Act of 2003.'
+ }
+ if self.original_mail:
+ headers['Reply-To'] = self.original_mail.origin_address
+ else:
+ headers['Reply-To'] = self.mailing_list.email_address
+
+ msg = EmailMultiAlternatives(subject=self.subject,
+ body=self.body,
+ to=to,
+ from_email=from_email,
+ headers=headers)
+ # Is this really the case? Right now it is, until we don't use the
+ # body or html_body and instead edit the MIME version itself.
+ msg.encoding = 'utf-8'
+ if self.html_body:
+ msg.attach_alternative(self.html_body, 'text/html')
+
+ conn = connection or self.mailing_list.get_smtp_connection()
+ conn.send_messages([msg])
+
+ if self.original_mail and self.original_mail.state != 'moderate':
+ self.original_mail.state = 'sent'
+ self.original_mail.save()
+
+ self.sent = datetime.now()
+ self.save()
class Meta:
ordering = ['-created']
View
29 interlink/tasks.py
@@ -1,23 +1,14 @@
-import traceback
-from datetime import datetime, date, timedelta
+from celery.task import task
-from front.scheduler import Task
+from models import MailingList, IncomingMail, OutgoingMail
-class EmailTask(Task):
- """A recurring task which checks the pending email records and either relays or moderates them."""
- def __init__(self, loopdelay=30, initdelay=5):
- Task.__init__(self, self.perform_task, loopdelay, initdelay)
- self.name = "EmailTask"
- def perform_task(self):
- from interlink.mail import DEFAULT_MAIL_CHECKER
- from models import MailingList, IncomingMail, OutgoingMail
- try:
- for mailing_list in MailingList.objects.all():
- DEFAULT_MAIL_CHECKER(mailing_list).fetch_mail()
- except:
- traceback.print_exc()
- IncomingMail.objects.process_incoming()
- OutgoingMail.objects.send_outgoing()
+@task(ignore_result=True)
+def email_task():
+ logger = email_task.get_logger()
+ MailingList.objects.fetch_all_mail(logger)
+ IncomingMail.objects.process_incoming()
+ OutgoingMail.objects.send_outgoing()
-# Copyright 2011 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+# Copyright 2012 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
View
54 interlink/tests/list_tests.py
@@ -1,21 +1,18 @@
-import traceback
from datetime import datetime, timedelta, date
-from django.conf import settings
from django.test import TestCase
-from django.core import management
-from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.core.management import call_command
+from django.core import mail
from staff.models import Member, MembershipPlan, Membership
from interlink.tests.test_utils import create_user
from interlink.models import MailingList, IncomingMail, OutgoingMail
-from interlink.mail import DEFAULT_MAIL_CHECKER, TestMailChecker, TEST_INCOMING_MAIL, add_test_incoming
+from interlink.mail import DEFAULT_MAIL_CHECKER, TestMailChecker, add_test_incoming
class ListTest(TestCase):
-
+
def setUp(self):
self.user1, self.client1 = create_user('alice', 'Alice', 'Dodgson', email='alice@example.com', is_staff=True)
self.user2, self.client2 = create_user('bob', 'Bob', 'Albert', email='bob@example.com')
@@ -24,9 +21,9 @@ def setUp(self):
email_address='hats@example.com', username='hat', password='1234',
pop_host='localhost', smtp_host='localhost'
)
-
+
self.basic_plan = MembershipPlan.objects.create(name='Basic', description='An occasional user', monthly_rate='50', daily_rate='25', dropin_allowance='5', deposit_amount='0')
-
+
def test_subscription_form(self):
Membership.objects.create(member=self.user2.get_profile(), membership_plan=self.basic_plan, start_date=date.today() - timedelta(days=10))
self.mlist1.moderators.add(self.user1)
@@ -41,17 +38,17 @@ def test_subscription_form(self):
self.assertEqual(OutgoingMail.objects.all().count(), 1)
IncomingMail.objects.process_incoming()
OutgoingMail.objects.send_outgoing()
-
+
def test_moderator_controlled(self):
self.assertEqual(0, self.mlist1.subscribers.count())
self.mlist1.moderator_controlled = True
self.mlist1.save()
self.mlist1.moderators.add(self.user1)
self.mlist1.subscribers.add(self.user2)
self.assertEqual(1, self.mlist1.subscribers.count())
-
+
checker = DEFAULT_MAIL_CHECKER(self.mlist1)
-
+
# check that non-moderator emails are rejected
add_test_incoming(self.mlist1, 'bob@example.com', 'ahoi 3', 'I like traffic lights.', sent_time=datetime.now() - timedelta(minutes=15))
incoming = checker.fetch_mail()
@@ -61,7 +58,7 @@ def test_moderator_controlled(self):
self.assertEqual(len(outgoing), 0)
income = IncomingMail.objects.get(pk=incoming[0].id)
self.assertEqual(income.state, 'reject')
-
+
add_test_incoming(self.mlist1, 'alice@example.com', 'ahoi 4', 'Who are you. Who who who who.', sent_time=datetime.now() - timedelta(minutes=10))
incoming = checker.fetch_mail()
self.assertEqual(len(incoming), 1)
@@ -70,53 +67,54 @@ def test_moderator_controlled(self):
self.assertEqual(len(outgoing), 1)
income = IncomingMail.objects.get(pk=incoming[0].id)
self.assertEqual(income.state, 'send')
-
+
def test_opt_out(self):
self.assertEqual(0, self.mlist1.subscribers.count())
self.mlist1.is_opt_out = True
self.mlist1.save()
- user3, client3 = create_user('suz', 'Suz', 'Ebens', email='suz@example.com')
+ user3, _client3 = create_user('suz', 'Suz', 'Ebens', email='suz@example.com')
self.assertEqual(0, self.mlist1.subscribers.count())
membership = Membership.objects.create(member=user3.get_profile(), membership_plan=self.basic_plan, start_date=date.today() - timedelta(days=31))
self.assertEqual(1, self.mlist1.subscribers.count())
self.assertTrue(user3 in self.mlist1.subscribers.all())
-
+
# Now test that subscribership isn't changed if a member is just changing to a new plan
membership.end_date = date.today() - timedelta(days=1)
membership.save()
self.mlist1.subscribers.remove(user3)
- membership2 = Membership.objects.create(member=user3.get_profile(), membership_plan=self.basic_plan, start_date=date.today())
+ _membership2 = Membership.objects.create(member=user3.get_profile(), membership_plan=self.basic_plan, start_date=date.today())
self.assertFalse(user3 in self.mlist1.subscribers.all())
def test_subscribe_command(self):
self.assertEqual(0, Member.objects.active_members().count())
self.assertEqual(0, self.mlist1.subscribers.count())
-
+
call_command('subscribe_members', '%s' % self.mlist1.id)
self.assertEqual(0, self.mlist1.subscribers.count())
-
+
Membership.objects.create(member=self.user2.get_profile(), membership_plan=self.basic_plan, start_date=date.today() - timedelta(days=10))
call_command('subscribe_members', '%s' % self.mlist1.id)
self.assertEqual(1, self.mlist1.subscribers.count())
-
-
+
def test_outgoing_processing(self):
self.assertEqual(OutgoingMail.objects.all().count(), 0)
OutgoingMail.objects.send_outgoing()
checker = DEFAULT_MAIL_CHECKER(self.mlist1)
-
+
self.mlist1.subscribers.add(self.user2)
add_test_incoming(self.mlist1, 'bob@example.com', 'ahoi 3', 'I like traffic lights.', sent_time=datetime.now() - timedelta(minutes=15))
incoming = checker.fetch_mail()
IncomingMail.objects.process_incoming()
outgoing = OutgoingMail.objects.all()[0]
self.assertEqual(outgoing.sent, None)
+ self.assertEqual(0, len(mail.outbox))
OutgoingMail.objects.send_outgoing()
incoming = IncomingMail.objects.get(pk=incoming[0].id)
outgoing = OutgoingMail.objects.all()[0]
self.assertNotEqual(outgoing.sent, None)
self.assertEqual(incoming.state, 'sent')
-
+ self.assertEqual(1, len(mail.outbox))
+
def test_incoming_processing(self):
checker = DEFAULT_MAIL_CHECKER(self.mlist1)
# send an email from an unknown address
@@ -131,7 +129,7 @@ def test_incoming_processing(self):
outgoing = OutgoingMail.objects.all()[0]
self.assertEqual(outgoing.original_mail, incoming)
self.assertTrue(outgoing.subject.startswith('Moderation Request'))
-
+
# send an email from a known address, but not a subscriber
add_test_incoming(self.mlist1, 'alice@example.com', 'ahoi 2', 'I like traffic lights.', sent_time=datetime.now() - timedelta(minutes=15))
incoming = checker.fetch_mail()
@@ -141,7 +139,7 @@ def test_incoming_processing(self):
self.assertEqual(OutgoingMail.objects.all().count(), 2)
incoming = IncomingMail.objects.get(pk=incoming[0].id)
self.assertEqual(incoming.state, 'moderate')
-
+
# send an email from a subscriber
self.mlist1.subscribers.add(self.user2)
add_test_incoming(self.mlist1, 'bob@example.com', 'ahoi 3', 'I like traffic lights.', sent_time=datetime.now() - timedelta(minutes=15))
@@ -154,7 +152,7 @@ def test_incoming_processing(self):
self.assertEqual(incoming.state, 'send')
outgoing = OutgoingMail.objects.all()[0]
self.assertTrue(outgoing.subject.startswith(self.mlist1.subject_prefix), outgoing.subject)
-
+
def test_recipients(self):
self.assertEqual(len(self.mlist1.subscriber_addresses), 0)
self.assertEqual(len(self.mlist1.moderator_addresses), 0)
@@ -164,7 +162,7 @@ def test_recipients(self):
self.mlist1.moderators.add(self.user1)
self.assertEqual(len(self.mlist1.subscriber_addresses), 1)
self.assertEqual(len(self.mlist1.moderator_addresses), 1)
-
+
def test_mail_checking(self):
self.assertEqual(DEFAULT_MAIL_CHECKER, TestMailChecker)
checker = DEFAULT_MAIL_CHECKER(self.mlist1)
@@ -174,9 +172,9 @@ def test_mail_checking(self):
self.assertEqual(len(in_mail), 1)
self.assertEqual(in_mail[0].origin_address, 'alice@example.com')
self.assertEqual(IncomingMail.objects.all().count(), 1)
-
+
add_test_incoming(self.mlist1, 'alice@example.com', 'ahoi 2', 'I like traffic lights A LOT.', sent_time=datetime.now() - timedelta(minutes=15))
- self.assertEqual(MailingList.objects.fetch_all_mail(), [])
+ MailingList.objects.fetch_all_mail()
self.assertEqual(IncomingMail.objects.all().count(), 2)
# Copyright 2011 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
View
5 local_settings.dist
@@ -32,11 +32,6 @@ EMAIL_USE_TLS = True
EMAIL_PORT = 587
EMAIL_SUBJECT_PREFIX = "[COWORKING] " # or None if you want no subject prefix
-from interlink.tasks import EmailTask
-from staff.tasks import MailingListTask, BillingTask
-# You can set the frequency of any Task by passing loopdelay=<number of seconds>
-SCHEDULED_TASKS = (EmailTask(), MailingListTask(), BillingTask())
-
import datetime
BILLING_START_DATE = datetime.date(2009, 11, 17)
NEW_MEMBER_DEPOSIT = 500
View
2 requirements.txt
@@ -5,3 +5,5 @@ pil
feedparser
django-taggit
django-taggit-templatetags
+celery==2.5.1
+django-celery==2.5.1
View
40 settings.py
@@ -2,6 +2,8 @@
import os
import sys
+from datetime import timedelta
+
PROJECT_ROOT = os.path.realpath(os.path.dirname(__file__))
TEMPLATE_DIRS = ( PROJECT_ROOT + '/templates/', )
MEDIA_ROOT = PROJECT_ROOT + '/media/'
@@ -100,6 +102,7 @@
'django.contrib.humanize',
'django.contrib.staticfiles',
'taggit',
+ 'djcelery',
'taggit_templatetags',
'south',
'front',
@@ -110,6 +113,43 @@
'tablet',
)
+#
+# Celery initialization
+#
+try:
+ import djcelery
+ djcelery.setup_loader()
+except ImportError:
+ pass
+
+BROKER_URL = "amqp://guest:guest@localhost:5672//"
+
+#
+# Celery beat schedules
+#
+CELERYBEAT_SCHEDULE = {
+ "email-task": {
+ "task": "interlink.tasks.email_task",
+ "schedule": timedelta(seconds=2),
+ },
+ #"billing-task": {
+ # "task": "staff.tasks.billing_task",
+ # "schedule": timedelta(hours=1)
+ #},
+ "unsubscribe-dropouts": {
+ "task": "staff.tasks.unsubscribe_recent_dropouts_task",
+ "schedule": timedelta(hours=1)
+ },
+}
+CELERY_DISABLE_RATE_LIMITS = True
+CELERY_RESULT_BACKEND = "amqp"
+
+# When this is True, celery tasks will be run synchronously.
+# This is nice when running unit tests or in development.
+# In production set this to False in your local_settings.py
+CELERY_ALWAYS_EAGER = False
+
+
from local_settings import *
# Copyright 2009, 2010 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
View
44 staff/tasks.py
@@ -1,37 +1,27 @@
-import traceback
-from datetime import datetime, date, timedelta
+from celery.task import task
-from django.core.exceptions import ObjectDoesNotExist
+from datetime import datetime, timedelta
-from front.scheduler import Task, ONE_DAY_SECONDS, ONE_HOUR_SECONDS
-class BillingTask(Task):
+@task(ignore_result=True)
+def billing_task():
"""
A recurring task which calculates billing.
The task runs once an hour, but will only run billing once every 24 hours.
"""
- def __init__(self, loopdelay=ONE_HOUR_SECONDS, initdelay=1):
- Task.__init__(self, self.perform_task, loopdelay, initdelay)
- self.name = "BillingRunTask"
-
- def perform_task(self):
- from staff import billing
- from staff.models import BillingLog
- try:
- latest_billing_log = BillingLog.objects.latest()
- if latest_billing_log.started > datetime.now() - timedelta(hours=24): return
- except ObjectDoesNotExist:
- pass
- billing.run_billing()
-
-class MailingListTask(Task):
+ import billing
+ from models import BillingLog
+
+ last_day = datetime.now() - timedelta(hours=24)
+ if not BillingLog.objects.filter(started__gt=last_day).exists():
+ billing.run_billing()
+
+
+@task(ignore_result=True)
+def unsubscribe_recent_dropouts_task():
"""A recurring task which checks for members who need to be unsubscribed from mailing lists"""
- def __init__(self, loopdelay=3600, initdelay=5):
- Task.__init__(self, self.perform_task, loopdelay, initdelay)
- self.name = "MailingListTask"
+ from models import Member
+ Member.objects.unsubscribe_recent_dropouts()
- def perform_task(self):
- from staff.models import Member
- Member.objects.unsubscribe_recent_dropouts()
-# Copyright 2011 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+# Copyright 2012 Office Nomads LLC (http://www.officenomads.com/) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

0 comments on commit ae6f685

Please sign in to comment.