From 46032246007260f0ffd5df1abb4a63072e629724 Mon Sep 17 00:00:00 2001 From: Stephen LaPorte Date: Mon, 7 Sep 2020 13:14:48 -0700 Subject: [PATCH 1/8] add support for sendy mailer --- requirements.txt | 24 ++++++++++------ weeklypedia/bake.py | 5 ++++ weeklypedia/mail.py | 31 +++++++++++++++++++- weeklypedia/publish.py | 34 ++++++++++++++++------ weeklypedia/secrets-default.json | 49 ++++++++++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 weeklypedia/secrets-default.json diff --git a/requirements.txt b/requirements.txt index 11f32f4..99a11e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,18 @@ -Werkzeug==0.9.4 -argparse==1.2.1 --e git+https://github.com/mahmoud/clastic.git#egg=clastic +ashes==0.7.6 +Babel==1.3 +certifi==2020.6.20 +chardet==3.0.4 +-e git+https://github.com/mahmoud/clastic.git@d6afb4ae849e8b463885465a01e28c1708d12a47#egg=clastic docopt==0.4.0 +idna==2.10 mailchimp==2.0.7 --e git+https://github.com/slaporte/wapiti.git#egg=wapiti -wsgiref==0.1.2 -cronfed==0.2.1 -ashes==0.7.6 -PyYAML==3.11 python-dateutil==2.4.2 -Babel==2.0 +pytz==2014.7 +PyYAML==3.11 +readline==6.2.4.1 +requests==2.24.0 +sendypy==4.0.9 +six==1.9.0 +twitter==1.14.3 +urllib3==1.25.10 +Werkzeug==0.9.4 diff --git a/weeklypedia/bake.py b/weeklypedia/bake.py index 17da9cc..baab176 100644 --- a/weeklypedia/bake.py +++ b/weeklypedia/bake.py @@ -80,6 +80,11 @@ def send(self, list_id, send_key): mailinglist.send_next_campaign() return 'Success: sent issue %s' % self.lang + def sendy_send(self, list_id, send_key): + sendy = SendyMailinglist() + sendy.create_and_send(self.subject, self.read_html(), self_read_text(), list_id) + return 'Success: sent issue %s via sendy' % self.lang + def get_past_issue_paths(lang, include_dev=False): ret = [] diff --git a/weeklypedia/mail.py b/weeklypedia/mail.py index bbecf65..394eb81 100644 --- a/weeklypedia/mail.py +++ b/weeklypedia/mail.py @@ -6,8 +6,13 @@ import mailchimp _CUR_PATH = os.path.dirname(os.path.abspath(__file__)) -KEY = json.load(open(os.path.join(_CUR_PATH, 'secrets.json'))).get('mc') +with open(os.path.join(_CUR_PATH, 'secrets.json')) as secrets_json: + secrets = json.load(secrets_json) + KEY = secrets.get('mc') + SENDY_KEY = secrets.get('sendy_key') + +SENDY_URL 'https://mailer.hatnote.com/sendy' TEST_LIST_ID = "a5ecbc7404" DEFAULT_LIST = TEST_LIST_ID @@ -51,6 +56,30 @@ def get_next_campaign(self): return campaigns['data'][0]['id'] +class SendyMailinglist(object): + def __init__(self): + self.key = key + self.client = SendyAPI(host=SENDY_URL, + api_key=SENDY_KEY) + + def create_and_send(self, + subject, + html_content, + text_content, + list_id): + resp = client.create_campaign( + from_name='Weeklypedia Digest', + from_email='weeklypedia@hatnote.com', + reply_to='weeklypedia@hatnote.com', + title=subject, + subject=subject, + plain_text=text_content, + html_text=html_content, + list_ids=list_id, + send_campaign=1, # if 0, then it will be saved as a draft. How do you send a draft campaign? + ) + print resp + if __name__ == '__main__': mc = Mailinglist(KEY) import pdb; pdb.set_trace() diff --git a/weeklypedia/publish.py b/weeklypedia/publish.py index 310233a..a9ca668 100644 --- a/weeklypedia/publish.py +++ b/weeklypedia/publish.py @@ -17,16 +17,32 @@ SUPPORTED_LANGS) _CUR_PATH = dirname(os.path.abspath(__file__)) -LIST_ID_MAP = json.load(open(os.path.join(_CUR_PATH, 'secrets.json'))).get('list_ids') +with open(os.path.join(_CUR_PATH, 'secrets.json')) as secrets_json: + secrets = json.load(secrets_json) + LIST_ID_MAP = secrets.get('list_ids') + SENDY_ID_MAP = secrets.get('sendy_ids') -def send_issue(lang, is_dev=False): - if is_dev: - list_id = DEBUG_LIST_ID - else: - list_id = LIST_ID_MAP[lang] + +def send_issue(lang, mailer, is_dev=False): cur_issue = Issue(lang, include_dev=is_dev) - return cur_issue.send(list_id, SENDKEY) + + if mailer == 'mailchimp': + if is_dev: + list_id = DEBUG_LIST_ID + else: + list_id = LIST_ID_MAP[lang] + result = cur_issue.send(list_id, SENDKEY) + elif mailer == 'sendy': + list_id = SENDY_ID_MAP[lang] + result = cur_issue.sendy_send(list_id) + return result + + +def send_send_issue(lang, is_dev=False): + list_id = SENDY_LIST_ID_MAP[lang] + cur_issue = + def get_argparser(): desc = 'Bake and send Weeklypedia issues. (Please fetch first)' @@ -34,6 +50,7 @@ def get_argparser(): prs.add_argument('--lang', default=None) prs.add_argument('--bake_all', default=False, action='store_true') prs.add_argument('--debug', default=DEBUG, action='store_true') + prs.add_argument('--mailer', default='mailchimp') return prs @@ -48,5 +65,6 @@ def get_argparser(): bake_latest_issue(issue_ashes_env, lang=lang, include_dev=debug) if args.lang in SUPPORTED_LANGS: lang = args.lang + mailer = args.mailer print bake_latest_issue(issue_ashes_env, lang=lang, include_dev=debug) - print send_issue(lang, debug) + print send_issue(lang, mailer, debug) diff --git a/weeklypedia/secrets-default.json b/weeklypedia/secrets-default.json new file mode 100644 index 0000000..247783d --- /dev/null +++ b/weeklypedia/secrets-default.json @@ -0,0 +1,49 @@ +{ + "mc": "...", + "sendy_key": "...", + "list_ids": { + "en": "...", + "de": "...", + "fr": "...", + "ko": "...", + "et": "...", + "sv": "...", + "it": "...", + "ca": "...", + "da": "...", + "es": "...", + "fa": "...", + "zh": "...", + "ur": "...", + "kn": "...", + "lv": "...", + "el": "...", + "te": "...", + "oc": "...", + "ru": "...", + "uk": "..." + }, + "sendy_ids": { + "en": "...", + "de": "...", + "fr": "...", + "ko": "...", + "et": "...", + "sv": "...", + "it": "...", + "ca": "...", + "da": "...", + "es": "...", + "fa": "...", + "zh": "...", + "ur": "...", + "kn": "...", + "lv": "...", + "el": "...", + "te": "...", + "oc": "...", + "ru": "...", + "uk": "..." + }, + "key": "..." +} From 145dccf575cfdef6980ae3eda775363f7167418d Mon Sep 17 00:00:00 2001 From: Stephen LaPorte Date: Mon, 7 Sep 2020 20:04:33 -0700 Subject: [PATCH 2/8] fixes and remove sendypy --- weeklypedia/bake.py | 7 +++---- weeklypedia/common.py | 45 +++++++++++++++++++++--------------------- weeklypedia/mail.py | 44 ++++++++++++++++++----------------------- weeklypedia/publish.py | 5 ----- 4 files changed, 45 insertions(+), 56 deletions(-) diff --git a/weeklypedia/bake.py b/weeklypedia/bake.py index baab176..df7f90f 100644 --- a/weeklypedia/bake.py +++ b/weeklypedia/bake.py @@ -10,7 +10,7 @@ from dateutil.parser import parse as parse_date from ashes import TemplateNotFound -from mail import Mailinglist, KEY +from mail import Mailinglist, sendy_send_campaign, KEY from fetch import get_latest_data_path from common import (DATA_BASE_PATH, @@ -80,9 +80,8 @@ def send(self, list_id, send_key): mailinglist.send_next_campaign() return 'Success: sent issue %s' % self.lang - def sendy_send(self, list_id, send_key): - sendy = SendyMailinglist() - sendy.create_and_send(self.subject, self.read_html(), self_read_text(), list_id) + def sendy_send(self, list_id): + sendy_send_campaign(self.subject, self.read_text(), self.read_html(), list_id) return 'Success: sent issue %s via sendy' % self.lang diff --git a/weeklypedia/common.py b/weeklypedia/common.py index ccb300c..f172aa6 100644 --- a/weeklypedia/common.py +++ b/weeklypedia/common.py @@ -15,29 +15,30 @@ _CUR_PATH = dirname(os.path.abspath(__file__)) LANG_MAP = json.load(open(pjoin(_CUR_PATH, 'language_codes.json'))) -LOCAL_LANG_MAP = {'en': u'English', - 'de': u'Deutsch', - 'fr': u'Français', - 'ko': u'한국어', - 'et': u'Eesti', - 'sv': u'Svenska', - 'da': u'Dansk', - 'it': u'Italiano', - 'ca': u'Català', - 'es': u'Español', - 'fa': u'فارسی', - 'ur': u'اردو', - 'zh': u'中文', - 'kn': u'ಕನ್ನಡ', - 'lv': u'Latvian', - 'el': u'ελληνική', - 'te': u'తెలుగు', - 'oc': 'Occitan', - 'ru': 'Russian', - 'uk': 'Ukrainian'} +LOCAL_LANG_MAP = {'en': u'English'} + # 'de': u'Deutsch', + # 'fr': u'Français', + # 'ko': u'한국어', + # 'et': u'Eesti', + # 'sv': u'Svenska', + # 'da': u'Dansk', + # 'it': u'Italiano', + # 'ca': u'Català', + # 'es': u'Español', + # 'fa': u'فارسی', + # 'ur': u'اردو', + # 'zh': u'中文', + # 'kn': u'ಕನ್ನಡ', + # 'lv': u'Latvian', + # 'el': u'ελληνική', + # 'te': u'తెలుగు', + # 'oc': 'Occitan', + # 'ru': 'Russian', + # 'uk': 'Ukrainian'} SENDKEY = json.load(open(os.path.join(_CUR_PATH, 'secrets.json'))).get('key') -SUPPORTED_LANGS = ['en', 'de', 'fr', 'ko', 'et', 'sv', 'da', 'it', 'ca', 'es', - 'fa', 'zh', 'ur', 'kn', 'lv', 'el', 'te', 'oc', 'ru', 'uk'] +SUPPORTED_LANGS = ['en'] +# 'de', 'fr', 'ko', 'et', 'sv', 'da', 'it', 'ca', 'es', +# 'fa', 'zh', 'ur', 'kn', 'lv', 'el', 'te', 'oc', 'ru', 'uk'] API_BASE_URL = 'http://weeklypedia.toolforge.org/fetch/' ARCHIVE_BASE_PATH = pjoin(dirname(_CUR_PATH), 'static', 'archive') diff --git a/weeklypedia/mail.py b/weeklypedia/mail.py index 394eb81..376eccf 100644 --- a/weeklypedia/mail.py +++ b/weeklypedia/mail.py @@ -3,6 +3,8 @@ import os import json +import requests + import mailchimp _CUR_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -12,13 +14,29 @@ KEY = secrets.get('mc') SENDY_KEY = secrets.get('sendy_key') -SENDY_URL 'https://mailer.hatnote.com/sendy' +SENDY_URL = 'http://mailer.hatnote.com/s/' TEST_LIST_ID = "a5ecbc7404" DEFAULT_LIST = TEST_LIST_ID # 'f811608f69' +def sendy_send_campaign(subject, text_content, html_content, list_id): + url = SENDY_URL + 'api/campaigns/create.php' + data = {'from_name': 'Weeklypedia Digest', + 'from_email': 'weeklypedia@hatnote.com', + 'reply_to': 'weeklypedia@hatnote.com', + 'title': subject, + 'subject': subject, + 'plain_text': text_content, + 'html_text': html_content, + 'list_ids': list_id, + 'send_campaign': 1, + 'api_key': SENDY_KEY} + resp = requests.post(url, data=data) + return resp + + class Mailinglist(object): def __init__(self, key): self.client = mailchimp.Mailchimp(key) @@ -56,30 +74,6 @@ def get_next_campaign(self): return campaigns['data'][0]['id'] -class SendyMailinglist(object): - def __init__(self): - self.key = key - self.client = SendyAPI(host=SENDY_URL, - api_key=SENDY_KEY) - - def create_and_send(self, - subject, - html_content, - text_content, - list_id): - resp = client.create_campaign( - from_name='Weeklypedia Digest', - from_email='weeklypedia@hatnote.com', - reply_to='weeklypedia@hatnote.com', - title=subject, - subject=subject, - plain_text=text_content, - html_text=html_content, - list_ids=list_id, - send_campaign=1, # if 0, then it will be saved as a draft. How do you send a draft campaign? - ) - print resp - if __name__ == '__main__': mc = Mailinglist(KEY) import pdb; pdb.set_trace() diff --git a/weeklypedia/publish.py b/weeklypedia/publish.py index a9ca668..6fcdae1 100644 --- a/weeklypedia/publish.py +++ b/weeklypedia/publish.py @@ -39,11 +39,6 @@ def send_issue(lang, mailer, is_dev=False): return result -def send_send_issue(lang, is_dev=False): - list_id = SENDY_LIST_ID_MAP[lang] - cur_issue = - - def get_argparser(): desc = 'Bake and send Weeklypedia issues. (Please fetch first)' prs = ArgumentParser(description=desc) From 17f42f8ce7afc4f04b4f856e9379b509e858feb3 Mon Sep 17 00:00:00 2001 From: Stephen LaPorte Date: Mon, 14 Sep 2020 08:54:05 -0700 Subject: [PATCH 3/8] update templates for sendy migration --- weeklypedia/bake.py | 6 +- weeklypedia/issue_templates/base/archive.html | 112 ++++++++------ weeklypedia/issue_templates/base/email.html | 23 +++ weeklypedia/issue_templates/base/email.txt | 6 + .../template_archive_index.html | 65 ++++---- .../issue_templates/template_index.html | 139 ++++++++---------- weeklypedia/publish.py | 8 +- 7 files changed, 203 insertions(+), 156 deletions(-) diff --git a/weeklypedia/bake.py b/weeklypedia/bake.py index df7f90f..8f86f57 100644 --- a/weeklypedia/bake.py +++ b/weeklypedia/bake.py @@ -36,6 +36,9 @@ ARCHIVE_INDEX_PATH_TMPL = ARCHIVE_BASE_PATH + '/{lang_shortcode}/index.html' ARCHIVE_PATH_TMPL = pjoin(ARCHIVE_BASE_PATH, ARCHIVE_PATH_TMPL) +with open(os.path.join(_CUR_PATH, 'secrets.json')) as secrets_json: + secrets = json.load(secrets_json) + SENDY_ID_MAP = secrets.get('sendy_ids') class Issue(object): def __init__(self, @@ -185,6 +188,7 @@ def render_issue(render_ctx, issue_ashes_env, lang = render_ctx['short_lang_name'] env = issue_ashes_env ctx = localize_data(render_ctx, lang) + ctx['list_id'] = SENDY_ID_MAP[lang] if format == 'html': ret = lang_fallback_render(env, lang, 'archive.html', ctx) elif format == 'email': @@ -236,7 +240,7 @@ def render_archive(issue_ashes_env, lang): ret['issues'].insert(0, {'path': archive_path, 'date': display_date}) ret['lang'] = LANG_MAP[lang] - ret['signup_url'] = SIGNUP_MAP[lang] + ret['list_id'] = SENDY_ID_MAP[lang] return issue_ashes_env.render('template_archive_index.html', ret) diff --git a/weeklypedia/issue_templates/base/archive.html b/weeklypedia/issue_templates/base/archive.html index 6f81a2f..78d53f4 100644 --- a/weeklypedia/issue_templates/base/archive.html +++ b/weeklypedia/issue_templates/base/archive.html @@ -12,57 +12,73 @@ $HEAD_TITLE$ -
-

$BODY_TITLE$

-

$DESCRIPTION$

-

$DELIVERY_DESCRIPTION$

-
-
- -
-
-
-

{intro|s}

-

$ARTICLES_HEADING$

-

{#stats}$ARTICLES_SUMMARY${/stats} $ARTICLES_INTRO$:

-
    - {#mainspace} -
  1. {title_s} ($DIFF_COUNT$ $DIFF_AUTHOR_COUNT$)
    2. - {/mainspace} -
+
+
+

$BODY_TITLE$

+

$DESCRIPTION$

+

$DELIVERY_DESCRIPTION$

+
+
+ +
+
+
+

{intro|s}

+

$ARTICLES_HEADING$

+

{#stats}$ARTICLES_SUMMARY${/stats} $ARTICLES_INTRO$:

+
    + {#mainspace} +
  1. {title_s} ($DIFF_COUNT$ $DIFF_AUTHOR_COUNT$)
    2. + {/mainspace} +
-

$NEW_ARTICLES_HEADING$

-

$NEW_ARTICLES_INTRO$

-
    - {#new_articles} -
  1. {title_s} ($DIFF_COUNT$ $DIFF_AUTHOR_COUNT$)
    2. - {/new_articles} -
+

$NEW_ARTICLES_HEADING$

+

$NEW_ARTICLES_INTRO$

+
    + {#new_articles} +
  1. {title_s} ($DIFF_COUNT$ $DIFF_AUTHOR_COUNT$)
    2. + {/new_articles} +
-

$DISCUSSIONS_HEADING$

-

$DISCUSSIONS_INTRO$:

-
    - {#talkspace} -
  1. {title_s}
    2. - {/talkspace} -
+

$DISCUSSIONS_HEADING$

+

$DISCUSSIONS_INTRO$:

+
    + {#talkspace} +
  1. {title_s}
    2. + {/talkspace} +
-

$STATS_HEADING$

-

$STATS_INTRO$:

-
    -
  • $USER_EDIT_COUNTS$
    • -
  • $ANON_EDIT_COUNTS$
    • -
  • $BOT_EDIT_COUNTS$
    • -
- -
-

$ABOUT_HEADING$

-

$ABOUT_TEXT$

-

$BLOG_LINK$.

+

$STATS_HEADING$

+

$STATS_INTRO$:

+
    +
  • $USER_EDIT_COUNTS$
    • +
  • $ANON_EDIT_COUNTS$
    • +
  • $BOT_EDIT_COUNTS$
    • +
+ +

$SIGNUP$

+

$PITCH$

+ +
+ + +
+
+
+ + + +
+ +
+

$ABOUT_HEADING$

+

$ABOUT_TEXT$

+

$BLOG_LINK$.

+
-
-

the WEEKLYPEDIA

-

A digest of the most-edited Wikipedia articles and discussions from the last week.

-

Delivered every Friday by email.

-
-
-

Next issue: {next_date} (Issue {volume})

-
-
-
-

Join a list to receive a weekly email with the top twenty Wikipedia articles and top five Wikipedia discussions from this weekk, available in English and more languages.

- +
+
+

the WEEKLYPEDIA

+

A digest of the most-edited Wikipedia articles and discussions from the last week.

+

Delivered every Friday by email.

+
+
+

Next issue: {next_date} (Issue {volume})

+
+
+
+

Join a list to receive a weekly email with the top twenty Wikipedia articles and top five Wikipedia discussions from this weekk, available in English and more languages.

+ - + -
-
-

Last week, in Issue {issue_number}: {stats.all_users|ci} users made {stats.edits|ci} changes to {stats.titles|ci} articles on English Wikipedia. The top articles for the week:

-
    - {#mainspace} -
  1. {title_s} ({edits|ci} changes by {users} authors)
    2. - {/mainspace} -
- Read the full issue +
- -
-

Read more...

- +
+
+

Last week, in Issue {issue_number}: {stats.all_users|ci} users made {stats.edits|ci} changes to {stats.titles|ci} articles on English Wikipedia. The top articles for the week:

+
    + {#mainspace} +
  1. {title_s} ({edits|ci} changes by {users} authors)
    2. + {/mainspace} +
+ Read the full issue +
+ +
+
+

About

+

This project retrieves a list of the week's recent changes using Wikimedia Tool Labs, and delivers emails using MailChimp. Code and more information available on GitHub. Built by Stephen LaPorte and Mahmoud Hashemi.

+

Learn more on the Hatnote blog.

-
-
-

About

-

This project retrieves a list of the week's recent changes using Wikimedia Tool Labs, and delivers emails using MailChimp. Code and more information available on GitHub. Built by Stephen LaPorte and Mahmoud Hashemi.

-

Learn more on the Hatnote blog.