Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixes bug 814647 - Automatically send emails to all users that crash.

  • Loading branch information...
commit 9ff392dcd1ad00df4ef15cded9e180572274832e 1 parent a7aaea3
Adrian Gaudebert AdrianGaudebert authored
1  requirements/prod.txt
View
@@ -7,5 +7,6 @@ lxml==2.3.4
psycopg2==2.4.5
simplejson==2.5.0
statsd==0.5.1
+suds==0.4
thrift==0.8.0
web.py==0.36
1  socorro/cron/crontabber.py
View
@@ -43,6 +43,7 @@
socorro.cron.jobs.matviews.TCBSBuildCronApp|1d|02:00
socorro.cron.jobs.matviews.ExplosivenessCronApp|1d|02:00
socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1h
+ socorro.cron.jobs.automatic_emails.AutomaticEmailsCronApp|1h
'''
147 socorro/cron/jobs/automatic_emails.py
View
@@ -0,0 +1,147 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+
+from configman import ConfigurationManager, Namespace
+
+from socorro.cron.base import PostgresBackfillCronApp
+from socorro.external.exacttarget import exacttarget
+
+
+SQL_REPORTS = """
+ SELECT DISTINCT r.email
+ FROM reports r
+ LEFT JOIN emails e ON r.email = e.email
+ WHERE r.date_processed > %(start_date)s
+ AND r.date_processed < %(end_date)s
+ AND r.email IS NOT NULL
+ AND e.last_sending < %(delayed_date)s
+ AND r.product IN %(products)s
+"""
+
+
+SQL_FIELDS = (
+ 'email',
+ # 'product',
+ # 'version',
+ # 'release_channel',
+)
+
+
+SQL_UPDATE = """
+ UPDATE emails
+ SET last_sending = %(last_sending)s
+ WHERE email = %(email)s
+"""
+
+
+class AutomaticEmailsCronApp(PostgresBackfillCronApp):
+ """Send an email to every user that crashes and gives us his or her email
+ address. """
+
+ app_name = 'automatic-emails'
+ app_version = '1.0'
+ app_description = 'Automatic Emails sent to users when they crash.'
+
+ required_config = Namespace()
+ required_config.add_option(
+ 'delay_between_emails',
+ default='7',
+ doc='Delay between two emails sent to the same user, in days. '
+ )
+ required_config.add_option(
+ 'restrict_products',
+ default=['Firefox'],
+ doc='List of products for which to send an email. '
+ )
+ required_config.add_option(
+ 'exacttarget_user',
+ default='',
+ doc='ExactTarget API user. '
+ )
+ required_config.add_option(
+ 'exacttarget_password',
+ default='',
+ doc='ExactTarget API password. '
+ )
+
+ def __init__(self, *args, **kwargs):
+ super(AutomaticEmailsCronApp, self).__init__(*args, **kwargs)
+ self.email_service = exacttarget.ExactTarget(
+ user=self.config.exacttarget_user,
+ pass_=self.config.exacttarget_password
+ )
+
+ def run(self, connection, run_datetime):
+ cursor = connection.cursor()
+
+ sql_params = {
+ 'start_date': run_datetime - datetime.timedelta(hours=1),
+ 'end_date': run_datetime,
+ 'delayed_date': run_datetime - datetime.timedelta(days=self.config.delay_between_emails),
+ 'products': tuple(self.config.restrict_products)
+ }
+
+ cursor.execute(SQL_REPORTS, sql_params)
+ for row in cursor.fetchall():
+ report = dict(zip(SQL_FIELDS, row))
+ # print report
+ self.send_email(report)
+ self.update_user(report, run_datetime, connection)
+
+ def send_email(self, report):
+ list_service = self.email_service.list()
+
+ try:
+ subscriber = list_service.get_subscriber(
+ report['email'],
+ None,
+ ['SubscriberKey']
+ )
+ subscriber_key = subscriber.SubscriberKey
+ except exacttarget.NewsletterException:
+ # subscriber does not exist, let's give it an ID
+ subscriber_key = report['email']
+
+ fields = {
+ 'EMAIL_ADDRESS_': report['email'],
+ 'EMAIL_FORMAT_': 'H',
+ 'TOKEN': subscriber_key
+ }
+ email_name = 'socorro_dev_test'
+ self.email_service.trigger_send(email_name, fields)
+
+ def update_user(self, report, sending_datetime, connection):
+ cursor = connection.cursor()
+ sql_params = {
+ 'email': report['email'],
+ 'last_sending': sending_datetime
+ }
+ cursor.execute(SQL_UPDATE, sql_params)
+
+
+# For testing only, to be removed!
+# if __name__ == '__main__':
+# config_source = {
+# 'exacttarget_user': '',
+# 'exacttarget_password': ''
+# }
+
+# config_manager = ConfigurationManager(
+# [AutomaticEmailsCronApp.get_required_config()],
+# app_name='',
+# app_version='',
+# app_description='',
+# values_source_list=[config_source]
+# )
+
+# with config_manager.context() as config:
+# emailer = AutomaticEmailsCronApp(config, '')
+
+# report = {
+# 'email': 'adrian.gaudebert@gmail.com',
+# }
+
+# print emailer.send_email(report)
0  socorro/external/exacttarget/__init__.py
View
No changes.
3,859 socorro/external/exacttarget/et-wsdl.txt
View
3,859 additions, 0 deletions not shown
358 socorro/external/exacttarget/exacttarget.py
View
@@ -0,0 +1,358 @@
+"""
+This library provides a Python interface for working with
+ExactTarget's SOAP API.
+
+To watch the SOAP requests:
+import logging
+logging.getLogger('suds.client').addHandler(logging.StreamHandler(sys.__stdout__))
+logging.getLogger('suds.client').setLevel(logging.DEBUG)
+
+Some test code:
+et = ExactTarget('<user>', '<pass>')
+et.data_ext().add_record('Master_Subscribers', ['TOKEN', 'EMAIL_ADDRESS_', 'CREATED_DATE_', 'MODIFIED_DATE_'], ['hello', 'jlong@mozilla.com', '2011-01-01', '2011-01-01'])
+et.trigger_send('WelcomeEmail', 'jlong@mozilla.com', 'hello', 'H')
+"""
+
+from functools import wraps
+import logging
+
+from suds import WebFault
+from suds.client import Client
+from suds.wsse import *
+
+
+logging.getLogger('suds.client').setLevel(logging.DEBUG)
+
+
+class UnauthorizedException(Exception):
+ """Failure to log into the email server."""
+ pass
+
+
+class NewsletterException(Exception):
+ """Error when trying to talk to the the email server."""
+ pass
+
+
+# This is just a cached version. The real URL is:
+# https://webservice.s4.exacttarget.com/etframework.wsdl
+#
+# The cached version has been stripped down to make suds run 1000x
+# faster. I deleted most of the fields in the TriggeredSendDefinition
+# and TriggeredSend objects that we don't use.
+WSDL_URL = 'file://%s/et-wsdl.txt' % os.path.dirname(os.path.abspath(__file__))
+
+
+def assert_status(obj):
+ """Make sure the returned status is OK"""
+ if obj.OverallStatus != 'OK':
+ if hasattr(obj, 'Results') and len(obj.Results) > 0:
+ res = obj.Results[0]
+
+ if hasattr(res, 'ErrorMessage') and res.ErrorMessage:
+ raise NewsletterException(res.ErrorMessage)
+ elif hasattr(res, 'ValueErrors') and res.ValueErrors:
+ # For some reason, the value errors array is inside an array
+ val_errs = res.ValueErrors[0]
+ if len(val_errs) > 0:
+ raise NewsletterException(val_errs[0].ErrorMessage)
+ elif hasattr(res, 'StatusCode') and res.StatusCode == 'Error':
+ raise NewsletterException(res.StatusMessage)
+ raise NewsletterException(obj.OverallStatus)
+
+
+def assert_result(obj):
+ """Make sure the returned object has a result"""
+ if not hasattr(obj, 'Results') or len(obj.Results) == 0:
+ raise NewsletterException('No results returned')
+
+
+def handle_fault(e):
+ """Handle an exception thrown by suds, and throw the appropriate
+ type of error"""
+
+ if hasattr(e, 'fault') and hasattr(e.fault, 'faultstring'):
+ # We have no fault code for a login failure, so check the
+ # string
+ if e.fault.faultstring.lower() == 'login failed':
+ raise UnauthorizedException(str(e))
+ raise NewsletterException(str(e))
+
+
+def logged_in(f):
+ """ Decorator to ensure the request will be authenticated """
+
+ @wraps(f)
+ def wrapper(inst, *args, **kwargs):
+ if not inst.client:
+ inst.client = Client(WSDL_URL)
+
+ security = Security()
+ token = UsernameToken(inst.user, inst.pass_)
+ security.tokens.append(token)
+ inst.client.set_options(wsse=security)
+ return f(inst, *args, **kwargs)
+ return wrapper
+
+
+class ExactTargetObject(object):
+
+ def __init__(self, user, pass_, client=None):
+ self.client = client
+ self.user = user
+ self.pass_ = pass_
+
+ def create(self, name):
+ return self.client.factory.create(name)
+
+
+class ExactTargetList(ExactTargetObject):
+
+ @logged_in
+ def add_subscriber(self, list_ids, fields, records):
+ list_ids = [list_ids] if isinstance(list_ids, int) else list_ids
+ records = [records] if isinstance(records[0], basestring) else records
+
+ subscribers = []
+
+ for record in records:
+ subscriber = self.create('Subscriber')
+
+ # Remove properties so that suds doesn't create them as
+ # empty fields in the SOAP request which will fail.
+ # You will see this throughout all this code.
+ del subscriber.EmailTypePreference
+ del subscriber.Status
+
+ for i, v in enumerate(record):
+ if fields[i] == 'email':
+ subscriber.EmailAddress = v
+ subscriber.SubscriberKey = v
+ elif fields[i] == 'format':
+ subscriber.EmailTypePreference = v
+ else:
+ attr = self.create('Attribute')
+ attr.Name = fields[i]
+ attr.Value = v
+ subscriber.Attributes.append(attr)
+
+ subscribers.append(subscriber)
+
+ lsts = []
+ for list_id in list_ids:
+ lst = self.create('SubscriberList')
+ lst.ID = list_id
+ del lst.Status
+ lsts.append(lst)
+
+ subscriber.Lists = lsts
+
+ opt = self.create('SaveOption')
+ opt.PropertyName = '*'
+ opt.SaveAction = 'UpdateAdd'
+
+ opts = self.create('UpdateOptions')
+ opts.SaveOptions.SaveOption = [opt]
+
+ try:
+ obj = self.client.service.Update(opts, [subscriber])
+ assert_status(obj)
+ except WebFault, e:
+ handle_fault(e)
+
+ @logged_in
+ def get_subscriber(self, email, list_id, fields):
+ req = self.create('RetrieveRequest')
+ req.ObjectType = 'Subscriber'
+ req.Properties = ['ID', 'EmailAddress', 'EmailTypePreference']
+
+ filter_ = self.create('SimpleFilterPart')
+ filter_.Value = email
+ filter_.SimpleOperator = 'equals'
+ filter_.Property = 'EmailAddress'
+
+ req.Filter = filter_
+ del req.Options
+
+ try:
+ obj = self.client.service.Retrieve(req)
+ assert_status(obj)
+ assert_result(obj)
+ except WebFault, e:
+ handle_fault(e)
+
+ record = obj.Results[0]
+ res = {}
+
+ for field in fields:
+ if field == 'email':
+ res['email'] = record.EmailAddress
+ else:
+ for attr in record.Attributes:
+ if attr.Name == field:
+ res[field] = attr.Value
+
+ return res
+
+ @logged_in
+ def get_lists_for_subscriber(self, emails):
+ emails = [emails] if isinstance(emails, basestring) else emails
+
+ req = self.create('RetrieveRequest')
+ req.ObjectType = 'ListSubscriber'
+ req.Properties = ['ListID']
+
+ filter_ = self.create('SimpleFilterPart')
+ filter_.Value = emails[0]
+ filter_.SimpleOperator = 'equals'
+ filter_.Property = 'SubscriberKey'
+ req.Filter = filter_
+
+ del req.Options
+
+ try:
+ obj = self.client.service.Retrieve(req)
+ assert_status(obj)
+ assert_result(obj)
+ except WebFault, e:
+ handle_fault(e)
+
+ lists = []
+ for res in obj.Results:
+ lists.append(res.ListID)
+
+ return lists
+
+
+class ExactTargetDataExt(ExactTargetObject):
+
+ @logged_in
+ def add_record(self, data_ids, fields, records):
+ data_ids = [data_ids] if isinstance(data_ids, basestring) else data_ids
+
+ objs = []
+ for id in data_ids:
+ obj = self.create('DataExtensionObject')
+ props = []
+
+ for i, v in enumerate(records):
+ prop = self.create('APIProperty')
+ prop.Name = fields[i]
+ prop.Value = v
+
+ props.append(prop)
+
+ obj.Properties.Property = props
+ obj.CustomerKey = id
+ objs.append(obj)
+
+ opt = self.create('SaveOption')
+ opt.PropertyName = '*'
+ opt.SaveAction = 'UpdateAdd'
+
+ rtype = self.create('RequestType')
+ opts = self.create('UpdateOptions')
+ opts.SaveOptions.SaveOption = [opt]
+
+ try:
+ obj = self.client.service.Update(opts, objs)
+ assert_status(obj)
+ except WebFault, e:
+ handle_fault(e)
+
+ @logged_in
+ def get_record(self, data_id, token, fields, field='TOKEN'):
+ req = self.create('RetrieveRequest')
+ req.ObjectType = 'DataExtensionObject[%s]' % data_id
+ req.Properties = fields
+
+ filter_ = self.create('SimpleFilterPart')
+ filter_.Value = token
+ filter_.SimpleOperator = 'equals'
+ filter_.Property = field
+ req.Filter = filter_
+
+ del req.Options
+
+ try:
+ obj = self.client.service.Retrieve(req)
+ assert_status(obj)
+ assert_result(obj)
+ except WebFault, e:
+ handle_fault(e)
+
+ return dict((p.Name, p.Value)
+ for p in obj.Results[0].Properties.Property)
+
+
+class ExactTarget(ExactTargetObject):
+
+ @logged_in
+ def list(self):
+ return ExactTargetList(self.user, self.pass_, self.client)
+
+ @logged_in
+ def data_ext(self):
+ return ExactTargetDataExt(self.user, self.pass_, self.client)
+
+ @logged_in
+ def trigger_send(self, send_name, fields):
+ send = self.create('TriggeredSend')
+ defn = send.TriggeredSendDefinition
+
+ status = self.create('TriggeredSendStatusEnum')
+ defn.Name = send_name
+ defn.CustomerKey = send_name
+ defn.TriggeredSendStatus = status.Active
+
+ sub = self.create('Subscriber')
+ sub.EmailAddress = fields.pop('EMAIL_ADDRESS_')
+ sub.SubscriberKey = fields['TOKEN']
+ sub.EmailTypePreference = ('HTML'
+ if fields['EMAIL_FORMAT_'] == 'H'
+ else 'Text')
+ del sub.Status
+
+ for k, v in fields.items():
+ attr = self.create('Attribute')
+ attr.Name = k
+ attr.Value = v
+ sub.Attributes.append(attr)
+
+ send.Subscribers = [sub]
+
+ rtype = self.create('RequestType')
+ opts = self.create('CreateOptions')
+
+ try:
+ obj = self.client.service.Create(opts, [send])
+ assert_status(obj)
+ assert_result(obj)
+ except WebFault, e:
+ handle_fault(e)
+
+ @logged_in
+ def trigger_send_sms(self, send_name, mobile_number):
+ send = self.create('SMSTriggeredSend')
+ send.Number = mobile_number
+ defn = send.SMSTriggeredSendDefinition
+ defn.Name = send_name
+ defn.CustomerKey = send_name
+
+ sub = self.create('Subscriber')
+ sub.SubscriberKey = mobile_number
+ sub.EmailTypePreference = 'Text'
+
+ del sub.Status
+
+ send.Subscriber = sub
+
+ rtype = self.create('RequestType')
+ opts = self.create('CreateOptions')
+
+ try:
+ obj = self.client.service.Create(opts, [send])
+ assert_status(obj)
+ assert_result(obj)
+ except WebFault, e:
+ handle_fault(e)
236 socorro/unittest/cron/jobs/test_automatic_emails.py
View
@@ -0,0 +1,236 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import json
+import mock
+import psycopg2
+import unittest
+from nose.plugins.attrib import attr
+
+from configman import ConfigurationManager
+
+from socorro.cron import crontabber
+from socorro.cron.jobs import automatic_emails
+from socorro.lib.datetimeutil import utc_now
+from ..base import TestCaseBase, DSN
+
+
+#==============================================================================
+@attr(integration='postgres') # for nosetests
+class TestFunctionalAutomaticEmails(TestCaseBase):
+
+ def setUp(self):
+ super(TestFunctionalAutomaticEmails, self).setUp()
+ # prep a fake table
+ assert 'test' in DSN['database.database_name']
+ self.dsn = ('host=%(database.database_host)s '
+ 'dbname=%(database.database_name)s '
+ 'user=%(database.database_user)s '
+ 'password=%(database.database_password)s' % DSN)
+ self.conn = psycopg2.connect(self.dsn)
+ cursor = self.conn.cursor()
+
+ now = utc_now() - datetime.timedelta(minutes=30)
+ last_month = now - datetime.timedelta(days=31)
+
+ cursor.execute("""
+ INSERT INTO reports
+ (uuid, email, product, version, release_channel, date_processed)
+ VALUES (
+ '1',
+ 'someone@example.com',
+ 'WaterWolf',
+ '20.0',
+ 'Release',
+ '%(now)s'
+ ), (
+ '2',
+ 'someoneelse@example.com',
+ 'WaterWolf',
+ '20.0',
+ 'Release',
+ '%(now)s'
+ ), (
+ '3',
+ 'anotherone@example.com',
+ 'WaterWolf',
+ '20.0',
+ 'Release',
+ '%(now)s'
+ )
+ """ % {'now': now})
+
+ # Let's insert a duplicate
+ cursor.execute("""
+ INSERT INTO reports
+ (uuid, email, product, version, release_channel, date_processed)
+ VALUES (
+ '10',
+ 'anotherone@example.com',
+ 'WaterWolf',
+ '20.0',
+ 'Release',
+ '%(now)s'
+ )
+ """ % {'now': now})
+
+ # And let's insert some invalid crashes
+ cursor.execute("""
+ INSERT INTO reports
+ (uuid, email, product, version, release_channel, date_processed)
+ VALUES (
+ '11',
+ null,
+ 'WaterWolf',
+ '20.0',
+ 'Release',
+ '%(now)s'
+ ), (
+ '12',
+ 'myemail@example.com',
+ 'WaterWolf',
+ '20.0',
+ 'Release',
+ '%(last_month)s'
+ ), (
+ '13',
+ 'menime@example.com',
+ 'WaterWolf',
+ '20.0',
+ 'Release',
+ '%(now)s'
+ )
+ """ % {'now': now, 'last_month': last_month})
+
+ cursor.execute("""
+ INSERT INTO emails (email, last_sending)
+ VALUES (
+ 'someone@example.com',
+ '%(last_month)s'
+ ), (
+ 'someoneelse@example.com',
+ '%(last_month)s'
+ ), (
+ 'anotherone@example.com',
+ '%(last_month)s'
+ ), (
+ 'menime@example.com',
+ '%(now)s'
+ )
+ """ % {'now': now, 'last_month': last_month})
+
+ self.conn.commit()
+
+ def tearDown(self):
+ super(TestFunctionalAutomaticEmails, self).tearDown()
+ self.conn.cursor().execute("""
+ TRUNCATE TABLE reports, emails CASCADE;
+ """)
+ self.conn.commit()
+ self.conn.close()
+
+ def _setup_config_manager(
+ self,
+ delay_between_emails=7,
+ exacttarget_user='',
+ exacttarget_password='',
+ restrict_products=['WaterWolf']
+ ):
+ _super = super(TestFunctionalAutomaticEmails, self)._setup_config_manager
+ extra_value_source = {
+ 'crontabber.class-AutomaticEmailsCronApp.delay_between_emails': delay_between_emails,
+ 'crontabber.class-AutomaticEmailsCronApp.exacttarget_user': exacttarget_user,
+ 'crontabber.class-AutomaticEmailsCronApp.exacttarget_password': exacttarget_password,
+ 'crontabber.class-AutomaticEmailsCronApp.restrict_products': restrict_products,
+ }
+
+ config_manager, json_file = _super(
+ 'socorro.cron.jobs.automatic_emails.AutomaticEmailsCronApp|1h',
+ extra_value_source=extra_value_source
+ )
+ return config_manager, json_file
+
+ def _setup_simple_config(self):
+ return ConfigurationManager(
+ [automatic_emails.AutomaticEmailsCronApp.get_required_config()],
+ values_source_list=[{
+ 'delay_between_emails': 7,
+ 'exacttarget_user': '',
+ 'exacttarget_password': '',
+ 'restrict_products': ['WaterWolf'],
+ }]
+ )
+
+ @mock.patch('socorro.external.exacttarget.exacttarget.ExactTarget')
+ def test_cron_job(self, exacttarget_mock):
+ (config_manager, json_file) = self._setup_config_manager()
+ with config_manager.context() as config:
+ tab = crontabber.CronTabber(config)
+ tab.run_all()
+
+ information = json.load(open(json_file))
+ print information
+ assert information['automatic-emails']
+ assert not information['automatic-emails']['last_error']
+ assert information['automatic-emails']['last_success']
+
+ self.assertEqual(exacttarget_mock.return_value.trigger_send.call_count, 3)
+
+ @mock.patch('socorro.external.exacttarget.exacttarget.ExactTarget')
+ def test_run(self, exacttarget_mock):
+ config_manager = self._setup_simple_config()
+ with config_manager.context() as config:
+ job = automatic_emails.AutomaticEmailsCronApp(config, '')
+ job.run(self.conn, utc_now())
+
+ self.assertEqual(exacttarget_mock.return_value.trigger_send.call_count, 3)
+
+ @mock.patch('socorro.external.exacttarget.exacttarget.ExactTarget')
+ def test_send_email(self, exacttarget_mock):
+ list_service_mock = exacttarget_mock.return_value.list.return_value
+ subscriber = list_service_mock.get_subscriber.return_value
+ subscriber.SubscriberKey = 'fake@example.com'
+
+ config_manager = self._setup_simple_config()
+ with config_manager.context() as config:
+ job = automatic_emails.AutomaticEmailsCronApp(config, '')
+
+ report = {
+ 'email': 'fake@example.com',
+ 'product': 'WaterWolf',
+ 'version': '20.0',
+ 'release_channel': 'Release',
+ }
+ job.send_email(report)
+
+ fields = {
+ 'EMAIL_ADDRESS_': report['email'],
+ 'EMAIL_FORMAT_': 'H',
+ 'TOKEN': report['email']
+ }
+ exacttarget_mock.return_value.trigger_send.assert_called_with(
+ 'socorro_dev_test',
+ fields
+ )
+
+ def test_update_user(self):
+ config_manager = self._setup_simple_config()
+ with config_manager.context() as config:
+ job = automatic_emails.AutomaticEmailsCronApp(config, '')
+
+ report = {
+ 'email': 'someone@example.com'
+ }
+ now = utc_now()
+ job.update_user(report, now, self.conn)
+
+ cursor = self.conn.cursor()
+ cursor.execute("""
+ SELECT last_sending FROM emails WHERE email=%(email)s
+ """, report)
+
+ self.assertEqual(cursor.rowcount, 1)
+ row = cursor.fetchone()
+ self.assertEqual(row[0], now)
Please sign in to comment.
Something went wrong with that request. Please try again.