Permalink
Browse files

recommit this patch: make most of the API calls async using Celery (b…

…ug 704314)
  • Loading branch information...
1 parent 66ace7e commit a6e8348de4c32d4d7e5f96f9dc49ddea83e174b9 @jlongster jlongster committed Dec 12, 2011
Showing with 225 additions and 177 deletions.
  1. +50 −0 apps/news/newsletters.py
  2. +128 −0 apps/news/tasks.py
  3. +34 −177 apps/news/views.py
  4. +13 −0 settings.py
View
50 apps/news/newsletters.py
@@ -0,0 +1,50 @@
+"""This data provides an official list of newsletters and tracks
+backend-specific data for working with them in the email provider.
+
+It's used to lookup the backend-specific newsletter name from a
+generic one passed by the user. This decouples the API from any
+specific email provider."""
+
+__all__ = ('newsletter_field', 'newsletter_name', 'newsletter_fields', 'newsletter_names')
+
+newsletters = {
+ 'mozilla-and-you': 'MOZILLA_AND_YOU',
+ 'mobile': 'ABOUT_MOBILE',
+ 'beta': 'FIREFOX_BETA_NEWS',
+ 'aurora': 'AURORA',
+ 'about-mozilla': 'ABOUT_MOZILLA',
+ 'drumbeat': 'DRUMBEAT_NEWS_GROUP',
+ 'addons': 'ABOUT_ADDONS',
+ 'hacks': 'ABOUT_HACKS',
+ 'labs': 'ABOUT_LABS',
+ 'qa-news': 'QA_NEWS',
+ 'student-reps': 'STUDENT_REPS',
+ 'about-standards': 'ABOUT_STANDARDS',
+ 'mobile-addon-dev': 'MOBILE_ADDON_DEV',
+ 'addon-dev': 'ADD_ONS',
+ 'join-mozilla': 'JOIN_MOZILLA',
+ 'mozilla-phone': 'MOZILLA_PHONE',
+ 'app-dev': 'APP_DEV'
+}
+
+
+def newsletter_field(name):
+ """Lookup the backend-specific field for the newsletter"""
+ return newsletters.get(name)
+
+
+def newsletter_name(field):
+ """Lookup the generic name for this newsletter field"""
+ for k, v in newsletters.iteritems():
+ if v == field:
+ return k
+
+
+def newsletter_names():
+ """Get a list of all the availble newsletters"""
+ return newsletters.keys()
+
+
+def newsletter_fields():
+ """Get a list of all the newsletter backend-specific fields"""
+ return newsletters.values()
View
128 apps/news/tasks.py
@@ -0,0 +1,128 @@
+import uuid
+from datetime import date
+from urllib2 import URLError
+
+from django.conf import settings
+from celery.task import task
+
+from responsys import Responsys, NewsletterException, UnauthorizedException
+from models import Subscriber
+from newsletters import *
+
+
+# A few constants to indicate the type of action to take
+# on a user with a list of newsletters
+SUBSCRIBE=1
+UNSUBSCRIBE=2
+SET=3
+
+
+def parse_newsletters(record, type, newsletters):
+ """Utility function to take a list of newsletters and according
+ the type of action (subscribe, unsubscribe, and set) set the
+ appropriate flags in `record` which is a dict of parameters that
+ will be sent to Responsys."""
+
+ newsletters = [x.strip() for x in newsletters.split(',')]
+
+ if type == SUBSCRIBE or type == SET:
+ # Subscribe the user to these newsletters
+ for nl in newsletters:
+ name = newsletter_field(nl)
+ if name:
+ record['%s_FLG' % name] = 'Y'
+ record['%s_DATE' % name] = date.today().strftime('%Y-%m-%d')
+
+
+ if type == UNSUBSCRIBE or type == SET:
+ # Unsubscribe the user to these newsletters
+ unsubs = newsletters
+
+ if type == SET:
+ # Unsubscribe to the inversion of these newsletters
+ subs = set(newsletters)
+ all = set(newsletter_names())
+ unsubs = all.difference(subs)
+
+ for nl in unsubs:
+ name = newsletter_field(nl)
+ if name:
+ record['%s_FLG' % name] = 'N'
+ record['%s_DATE' % name] = date.today().strftime('%Y-%m-%d')
+
+
+@task(default_retry_delay=60) # retry in 1 minute on failure
+def update_user(data, authed_email, type):
+ """Task for updating user's preferences and newsletters.
+
+ ``authed_email`` is the email for the user pulled from the database
+ with their token, if exists."""
+
+ log = update_user.get_logger()
+
+ # Validate parameters
+ if not authed_email and 'email' not in data:
+ log.error('No user or email provided')
+
+ # Parse the parameters
+ record = {'EMAIL_ADDRESS_': data['email'],
+ 'EMAIL_PERMISSION_STATUS_': 'I'}
+
+ extra_fields = {
+ 'format': 'EMAIL_FORMAT_',
+ 'country': 'COUNTRY_',
+ 'lang': 'LANGUAGE_ISO2',
+ 'locale': 'LANG_LOCALE',
+ 'source_url': 'SOURCE_URL'
+ }
+
+ # Optionally add more fields
+ for field in extra_fields.keys():
+ if field in data:
+ record[extra_fields[field]] = data[field]
+
+ # Set the newsletter flags in the record
+ parse_newsletters(record, type, data.get('newsletters', ''))
+
+ # Get the user or create them
+ (sub, created) = Subscriber.objects.get_or_create(email=record['EMAIL_ADDRESS_'])
+
+ # Update the token if it's a new user or they aren't simply
+ # subscribing from a newsletter form (tokens are one-time use)
+ if created or type != SUBSCRIBE:
+ sub.token = str(uuid.uuid4())
+ record['TOKEN'] = sub.token
+ sub.save()
+
+ # Submit the final data to responsys
+ try:
+ rs = Responsys()
+ rs.login(settings.RESPONSYS_USER, settings.RESPONSYS_PASS)
+
+ if authed_email and record['EMAIL_ADDRESS_'] != authed_email:
+ # Email has changed, we need to delete the previous user
+ rs.delete_list_members(authed_email,
+ settings.RESPONSYS_FOLDER,
+ settings.RESPONSYS_LIST)
+
+ rs.merge_list_members(settings.RESPONSYS_FOLDER,
+ settings.RESPONSYS_LIST,
+ record.keys(),
+ record.values())
+
+ # Trigger the welcome event unless it is suppressed
+ if data.get('trigger_welcome', False) == 'Y':
+ rs.trigger_custom_event(record['EMAIL_ADDRESS_'],
+ settings.RESPONSYS_FOLDER,
+ settings.RESPONSYS_LIST,
+ 'New_Signup_Welcome')
+
+ rs.logout()
+ except URLError, e:
+ # URL timeout, try again
+ update_user.retry(exc=e)
+ except NewsletterException, e:
+ log.error('NewsletterException: %s' % e.message)
+ except UnauthorizedException, e:
+ log.error('Responsys auth failure')
+
View
211 apps/news/views.py
@@ -1,54 +1,21 @@
from functools import wraps
-from datetime import date
import urlparse
import json
-import uuid
-from django.http import (HttpResponse, HttpResponseRedirect,
- HttpResponseBadRequest, HttpResponseForbidden)
+from django.http import HttpResponse, HttpResponseBadRequest
from django.views.decorators.csrf import csrf_exempt
from django.conf import settings
+from tasks import update_user, SUBSCRIBE, UNSUBSCRIBE, SET
+from newsletters import *
from models import Subscriber
from responsys import Responsys, NewsletterException, UnauthorizedException
-NEWSLETTERS = {
- 'mozilla-and-you': 'MOZILLA_AND_YOU',
- 'mobile': 'ABOUT_MOBILE',
- 'beta': 'FIREFOX_BETA_NEWS',
- 'aurora': 'AURORA',
- 'about-mozilla': 'ABOUT_MOZILLA',
- 'drumbeat': 'DRUMBEAT_NEWS_GROUP',
- 'addons': 'ABOUT_ADDONS',
- 'hacks': 'ABOUT_HACKS',
- 'labs': 'ABOUT_LABS',
- 'qa-news': 'QA_NEWS',
- 'student-reps': 'STUDENT_REPS',
- 'about-standards': 'ABOUT_STANDARDS',
- 'mobile-addon-dev': 'MOBILE_ADDON_DEV',
- 'addon-dev': 'ADD_ONS',
- 'join-mozilla': 'JOIN_MOZILLA',
- 'mozilla-phone': 'MOZILLA_PHONE',
- 'app-dev': 'APP_DEV'
-}
-
-NEWSLETTER_NAMES = NEWSLETTERS.keys()
-NEWSLETTER_FIELDS = NEWSLETTERS.values()
-
-# Utility functions
-
-def newsletter_field(name):
- return NEWSLETTERS.get(name, False)
-
-
-def newsletter_name(field):
- i = NEWSLETTER_FIELDS.index(field)
- return NEWSLETTER_NAMES[i]
-
+## Utility functions
def logged_in(f):
- """ Decorator to check if the user has permission to view these
- pages """
+ """Decorator to check if the user has permission to view these
+ pages"""
@wraps(f)
def wrapper(request, token, *args, **kwargs):
@@ -66,49 +33,62 @@ def json_response(data, status=200):
res = HttpResponse(json.dumps(data),
mimetype='application/json')
res.status_code = status
-
- # Allow all cross-domain requests, this service will restrict
- # access on the server level
- res['Access-Control-Allow-Origin'] = '*'
return res
-class Update(object):
- SUBSCRIBE=1
- UNSUBSCRIBE=2
- SET=3
+def update_user_task(request, type, data=None):
+ """Call the update_user task async with the right parameters"""
+ user = getattr(request, 'subscriber', None)
+ update_user.apply_async((data or request.POST.copy(),
+ user and user.email,
+ type))
+
+## Views
@csrf_exempt
def subscribe(request):
+ if request.method != 'POST':
+ return HttpResponseBadRequest("Only POST supported")
+
if 'newsletters' not in request.POST:
return json_response({'desc': 'newsletters is missing'},
status=500)
- return update_user(request, Update.SUBSCRIBE)
+ # If the user isn't opting in yet, we tell the system to
+ # unsubscribe them instead of subscribe them, which sets a flag
+ # for those newsletters that still requires confirmation by other
+ # means (confirmation email, etc)
+ optin = request.POST.get('optin', 'Y') == 'Y'
+ update_user_task(request,
+ SUBSCRIBE if optin else UNSUBSCRIBE)
+ return json_response({})
@logged_in
@csrf_exempt
def unsubscribe(request, token):
+ if request.method != 'POST':
+ return HttpResponseBadRequest("Only POST supported")
+
data = request.POST.copy()
if data.get('optout', 'N') == 'Y':
data['optin'] = 'N'
+ data['newsletters'] = ','.join(newsletter_names())
- for field in NEWSLETTER_FIELDS:
- data['newsletters'] = ','.join(NEWSLETTER_NAMES)
-
- return update_user(request, Update.UNSUBSCRIBE, data)
+ update_user_task(request, UNSUBSCRIBE, data)
+ return json_response({})
@logged_in
@csrf_exempt
def user(request, token):
if request.method == 'POST':
- return update_user(request, Update.SET)
+ update_user_task(request, SET)
+ return json_response({})
- newsletters = NEWSLETTERS.values()
+ newsletters = newsletter_fields()
fields = [
'EMAIL_ADDRESS_',
@@ -148,129 +128,6 @@ def user(request, token):
return json_response(user_data)
-def parse_newsletters(record, type, newsletters, optout):
- """ Parse the newsletter data from a comma-delimited string and
- set the appropriate fields in the record """
-
- newsletters = [x.strip() for x in newsletters.split(',')]
-
- if type == Update.SUBSCRIBE or type == Update.SET:
- # Subscribe the user to these newsletters
- for nl in newsletters:
- name = newsletter_field(nl)
- if name:
- record['%s_FLG' % name] = 'N' if optout else 'Y'
- record['%s_DATE' % name] = date.today().strftime('%Y-%m-%d')
-
-
- if type == Update.UNSUBSCRIBE or type == Update.SET:
- # Unsubscribe the user to these newsletters
- unsubs = newsletters
-
- if type == Update.SET:
- # Unsubscribe to the inversion of these newsletters
- subs = set(newsletters)
- all = set(NEWSLETTER_NAMES)
- unsubs = all.difference(subs)
-
- for nl in unsubs:
- name = newsletter_field(nl)
- if name:
- record['%s_FLG' % name] = 'N'
-
-
-def update_user(request, type, data=None):
- """ General method for updating user's preferences and subscribed
- newsletters. Assumes data to be in POST """
-
- if request.method != 'POST':
- return HttpResponseBadRequest("Only POST supported")
-
- data = data or request.POST
- has_auth = hasattr(request, 'subscriber')
-
- # validate parameters
- if not has_auth and 'email' not in data:
- return json_response({'desc': 'email is required when not using tokens'},
- status=500)
-
- # parse the parameters
- record = {'EMAIL_ADDRESS_': data['email'],
- 'EMAIL_PERMISSION_STATUS_': 'I'}
-
- extra_fields = {
- 'format': 'EMAIL_FORMAT_',
- 'country': 'COUNTRY_',
- 'lang': 'LANGUAGE_ISO2',
- 'locale': 'LANG_LOCALE',
- 'source_url': 'SOURCE_URL'
- }
-
- # optionally add more fields
- for field in extra_fields.keys():
- if field in data:
- record[extra_fields[field]] = data[field]
-
- # setup the newsletter fields
- parse_newsletters(record,
- type,
- data.get('newsletters', ''),
- data.get('optin', 'Y') != 'Y')
-
- # make a new token
- token = str(uuid.uuid4())
-
- if type == Update.SUBSCRIBE:
- # if we are subscribing and the user already exists, don't
- # update the token. otherwise create a new user with the token.
- try:
- sub = Subscriber.objects.get(email=record['EMAIL_ADDRESS_'])
- token = sub.token
- except Subscriber.DoesNotExist:
- sub = Subscriber(email=record['EMAIL_ADDRESS_'], token=token)
- sub.save()
- else:
- # if we are updating an existing user, set a new token
- sub = Subscriber.objects.get(email=request.subscriber.email)
- sub.token = token
- sub.save()
-
- record['TOKEN'] = token
-
- # save the user's fields
- try:
- rs = Responsys()
- rs.login(settings.RESPONSYS_USER, settings.RESPONSYS_PASS)
-
- if has_auth and record['EMAIL_ADDRESS_'] != request.subscriber.email:
- # email has changed, we need to delete the previous user
- rs.delete_list_members(request.subscriber.email,
- settings.RESPONSYS_FOLDER,
- settings.RESPONSYS_LIST)
-
- rs.merge_list_members(settings.RESPONSYS_FOLDER,
- settings.RESPONSYS_LIST,
- record.keys(),
- record.values())
-
- if data.get('trigger_welcome', False) == 'Y':
- rs.trigger_custom_event(record['EMAIL_ADDRESS_'],
- settings.RESPONSYS_FOLDER,
- settings.RESPONSYS_LIST,
- 'New_Signup_Welcome')
-
- rs.logout()
- except NewsletterException, e:
- return json_response({'desc': e.message},
- status=500)
- except UnauthorizedException, e:
- return json_response({'desc': 'Responsys auth failure'},
- status=500)
-
-
- return json_response({'token': token})
-
-
@logged_in
@csrf_exempt
def delete_user(request, token):
View
13 settings.py
@@ -78,6 +78,7 @@
'fixture_magic',
'piston',
'tower',
+ 'djcelery',
'django.contrib.auth',
'django.contrib.contenttypes',
@@ -146,3 +147,15 @@ def JINJA_CONFIG():
RESPONSYS_PASS = ''
RESPONSYS_FOLDER = '!MasterData'
RESPONSYS_LIST = 'TEST_CONTACTS_LIST'
+
+# Uncomment these to use Celery, use eager for local dev
+# CELERY_ALWAYS_EAGER = False
+# BROKER_HOST = 'localhost'
+# BROKER_PORT = 5672
+# BROKER_USER = 'basket'
+# BROKER_PASSWORD = 'basket'
+# BROKER_VHOST = '/'
+# CELERY_RESULT_BACKEND = 'amqp'
+
+import djcelery
+djcelery.setup_loader()

0 comments on commit a6e8348

Please sign in to comment.