Browse files

initial commit

  • Loading branch information...
0 parents commit c4678dfac9dbe289be22fe2f1ceb7c1feeeaab34 @peterbe committed Sep 2, 2011
Showing with 2,852 additions and 0 deletions.
  1. +8 −0 README.md
  2. +23 −0 setup.py
  3. +1 −0 tornado_utils/__init__.py
  4. +48 −0 tornado_utils/decorators.py
  5. +34 −0 tornado_utils/edit_distance.py
  6. +22 −0 tornado_utils/git.py
  7. +28 −0 tornado_utils/goo_gl.py
  8. +491 −0 tornado_utils/html2text.py
  9. +84 −0 tornado_utils/http_test_client.py
  10. +49 −0 tornado_utils/routes.py
  11. +1 −0 tornado_utils/send_mail/__init__.py
  12. +1 −0 tornado_utils/send_mail/backends/__init__.py
  13. +41 −0 tornado_utils/send_mail/backends/base.py
  14. +34 −0 tornado_utils/send_mail/backends/console.py
  15. +26 −0 tornado_utils/send_mail/backends/locmem.py
  16. +54 −0 tornado_utils/send_mail/backends/pickle.py
  17. +106 −0 tornado_utils/send_mail/backends/smtp.py
  18. +13 −0 tornado_utils/send_mail/config.py
  19. +30 −0 tornado_utils/send_mail/dns_name.py
  20. +36 −0 tornado_utils/send_mail/importlib.py
  21. +57 −0 tornado_utils/send_mail/pickled_messages/20110710_163407_0.pickle
  22. +57 −0 tornado_utils/send_mail/pickled_messages/20110801_112712_0.pickle
  23. +57 −0 tornado_utils/send_mail/pickled_messages/20110806_150814_0.pickle
  24. +57 −0 tornado_utils/send_mail/pickled_messages/20110806_150826_0.pickle
  25. +57 −0 tornado_utils/send_mail/pickled_messages/20110806_150852_0.pickle
  26. +57 −0 tornado_utils/send_mail/pickled_messages/20110806_150906_0.pickle
  27. +57 −0 tornado_utils/send_mail/pickled_messages/20110806_150910_0.pickle
  28. +432 −0 tornado_utils/send_mail/send_email.py
  29. +42 −0 tornado_utils/stopwords.py
  30. +129 −0 tornado_utils/thumbnailer.py
  31. +134 −0 tornado_utils/timesince.py
  32. +413 −0 tornado_utils/tornado_static.py
  33. +11 −0 tornado_utils/truncate.py
  34. +162 −0 tornado_utils/utils.py
8 README.md
@@ -0,0 +1,8 @@
+tornado-utils
+=============
+
+A bunch of Tornado specific python utilities originally used on
+Kwissle.com
+
+(c) Peter Bengtsson, 2011
+
23 setup.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+import os
+from distutils.core import setup
+
+def read(fname):
+ return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+setup(name='tornado-utils',
+ version='1.0',
+ description='Utility scripts for a Tornado site',
+ long_description=read('README.md'),
+ author='Peter Bengtsson',
+ author_email='mail@peterbe.com',
+ url='http://github.com/peterbe/tornado-utils',
+ classifiers=[
+ 'Programming Language :: Python :: 2',
+ 'Intended Audience :: Developers',
+ 'Operating System :: POSIX :: Linux',
+ 'Topic :: Software Development :: Testing'
+ 'Topic :: Software Development :: Build Tools'
+ ],
+)
1 tornado_utils/__init__.py
@@ -0,0 +1 @@
+from utils import *
48 tornado_utils/decorators.py
@@ -0,0 +1,48 @@
+from urllib import quote as url_quote
+from tornado.web import HTTPError
+
+def login_required(func, redirect_to=None):
+ def is_logged_in(self):
+ guid = self.get_secure_cookie('user')
+ if guid:
+ if self.db.users.User(dict(guid=guid)):
+ return func(self)
+ if redirect_to:
+ next = self.request.path
+ if self.request.query:
+ next += '?%s' % self.request.query
+ url = redirect_to + '?next=%s' % url_quote(next)
+ self.redirect(url)
+ else:
+ raise HTTPError(403, "Must be logged in")
+ return is_logged_in
+
+def login_redirect(func):
+ return login_required(func, redirect_to='/login/')
+
+import functools
+import urllib
+import urlparse
+# taken from tornado.web.authenticated
+
+def authenticated_plus(extra_check):
+ """Decorate methods with this to require that the user be logged in."""
+ def wrap(method):
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if not (self.current_user and extra_check(self.current_user)):
+ if self.request.method in ("GET", "HEAD"):
+ url = self.get_login_url()
+ if "?" not in url:
+ if urlparse.urlsplit(url).scheme:
+ # if login url is absolute, make next absolute too
+ next_url = self.request.full_url()
+ else:
+ next_url = self.request.uri
+ url += "?" + urllib.urlencode(dict(next=next_url))
+ self.redirect(url)
+ return
+ raise HTTPError(403)
+ return method(self, *args, **kwargs)
+ return wrapper
+ return wrap
34 tornado_utils/edit_distance.py
@@ -0,0 +1,34 @@
+class EditDistance(object):
+ def __init__(self, against, alphabet=u'abcdefghijklmnopqrstuvwxyz'):
+ self.against = against
+ self.alphabet = alphabet
+
+ def match(self, word):
+ return list(self._match(word))
+
+ def _match(self, word):
+ for w in self._edits1(word):
+ if w in self.against:
+ yield w
+
+ def _edits1(self, word):
+ n = len(word)
+ return set(# deletion
+ [word[0:i]+word[i+1:] for i in range(n)] +
+ # transposition
+ [word[0:i]+word[i+1]+word[i]+word[i+2:] for i in range(n-1)] +
+ # alteration
+ [word[0:i]+c+word[i+1:] for i in range(n) for c in self.alphabet] +
+ # insertion
+ [word[0:i]+c+word[i:] for i in range(n+1) for c in self.alphabet])
+
+if __name__ == '__main__':
+ against = ('peter',)
+ ed = EditDistance(against)
+ assert ed.match('peter') == ['peter']
+ assert ed.match('petter') == ['peter']
+ assert ed.match('peffeer') == []
+
+ against = ('peter','petter')
+ ed = EditDistance(against)
+ assert ed.match('pettere') == ['petter']
22 tornado_utils/git.py
@@ -0,0 +1,22 @@
+import os, re
+import logging
+from subprocess import Popen, PIPE
+
+def get_git_revision():
+ return _get_git_revision()
+
+def _get_git_revision():
+ # this is actually very fast. Takes about 0.01 seconds on my machine!
+ home = os.path.dirname(__file__)
+ proc = Popen('cd %s;git log --no-color -n 1 --date=iso' % home,
+ shell=True, stdout=PIPE, stderr=PIPE)
+ output = proc.communicate()
+ try:
+ date = [x.split('Date:')[1].split('+')[0].strip() for x in
+ output[0].splitlines() if x.startswith('Date:')][0]
+ date_wo_tz = re.split('-\d{4}', date)[0].strip()
+ return date_wo_tz
+ except IndexError:
+ logging.debug("OUTPUT=%r" % output[0], exc_info=True)
+ logging.debug("ERROR=%r" % output[1])
+ return 'unknown'
28 tornado_utils/goo_gl.py
@@ -0,0 +1,28 @@
+import json
+import urllib
+import urllib2
+import logging
+# http://code.google.com/apis/urlshortener/v1/getting_started.html
+# This works:
+# curl https://www.googleapis.com/urlshortener/v1/url \
+# -H 'Content-Type: application/json'\
+# -d '{"longUrl": "http://www.peterbe.com"}'
+#
+URL = 'https://www.googleapis.com/urlshortener/v1/url'
+
+def shorten(url):
+ #data = urllib.urlencode({'longUrl': url})
+ data = json.dumps({'longUrl': url})
+ headers = {'Content-Type': 'application/json'}
+ req = urllib2.Request(URL, data, headers)
+ response = urllib2.urlopen(req)
+ the_page = response.read()
+ logging.info("Shorten %r --> %s" % (url, the_page))
+ struct = json.loads(the_page)
+ return struct['id']
+
+if __name__ == '__main__':
+ import sys
+ for url in sys.argv[1:]:
+ if '://' in url:
+ print shorten(url)
491 tornado_utils/html2text.py
@@ -0,0 +1,491 @@
+#!/usr/bin/env python
+"""html2text: Turn HTML into equivalent Markdown-structured text."""
+__version__ = "3.02"
+__author__ = "Aaron Swartz (me@aaronsw.com)"
+__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3."
+__contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
+
+# TODO:
+# Support decoded entities with unifiable.
+
+try:
+ True
+except NameError:
+ setattr(__builtins__, 'True', 1)
+ setattr(__builtins__, 'False', 0)
+
+def has_key(x, y):
+ if hasattr(x, 'has_key'): return x.has_key(y)
+ else: return y in x
+
+try:
+ import htmlentitydefs
+ import urlparse
+ import HTMLParser
+except ImportError: #Python3
+ import html.entities as htmlentitydefs
+ import urllib.parse as urlparse
+ import html.parser as HTMLParser
+try: #Python3
+ import urllib.request as urllib
+except:
+ import urllib
+import optparse, re, sys, codecs, types
+
+try: from textwrap import wrap
+except: pass
+
+# Use Unicode characters instead of their ascii psuedo-replacements
+UNICODE_SNOB = 0
+
+# Put the links after each paragraph instead of at the end.
+LINKS_EACH_PARAGRAPH = 0
+
+# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
+BODY_WIDTH = 78
+
+# Don't show internal links (href="#local-anchor") -- corresponding link targets
+# won't be visible in the plain text file anyway.
+SKIP_INTERNAL_LINKS = False
+
+### Entity Nonsense ###
+
+def name2cp(k):
+ if k == 'apos': return ord("'")
+ if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3
+ return htmlentitydefs.name2codepoint[k]
+ else:
+ k = htmlentitydefs.entitydefs[k]
+ if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1
+ return ord(codecs.latin_1_decode(k)[0])
+
+unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"',
+'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*',
+'ndash':'-', 'oelig':'oe', 'aelig':'ae',
+'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a',
+'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e',
+'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i',
+'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o',
+'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'}
+
+unifiable_n = {}
+
+for k in unifiable.keys():
+ unifiable_n[name2cp(k)] = unifiable[k]
+
+def charref(name):
+ if name[0] in ['x','X']:
+ c = int(name[1:], 16)
+ else:
+ c = int(name)
+
+ if not UNICODE_SNOB and c in unifiable_n.keys():
+ return unifiable_n[c]
+ else:
+ try:
+ return unichr(c)
+ except NameError: #Python3
+ return chr(c)
+
+def entityref(c):
+ if not UNICODE_SNOB and c in unifiable.keys():
+ return unifiable[c]
+ else:
+ try: name2cp(c)
+ except KeyError: return "&" + c + ';'
+ else:
+ try:
+ return unichr(name2cp(c))
+ except NameError: #Python3
+ return chr(name2cp(c))
+
+def replaceEntities(s):
+ s = s.group(1)
+ if s[0] == "#":
+ return charref(s[1:])
+ else: return entityref(s)
+
+r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
+def unescape(s):
+ return r_unescape.sub(replaceEntities, s)
+
+### End Entity Nonsense ###
+
+def onlywhite(line):
+ """Return true if the line does only consist of whitespace characters."""
+ for c in line:
+ if c is not ' ' and c is not ' ':
+ return c is ' '
+ return line
+
+def optwrap(text):
+ """Wrap all paragraphs in the provided text."""
+ if not BODY_WIDTH:
+ return text
+
+ assert wrap, "Requires Python 2.3."
+ result = ''
+ newlines = 0
+ for para in text.split("\n"):
+ if len(para) > 0:
+ if para[0] != ' ' and para[0] != '-' and para[0] != '*':
+ for line in wrap(para, BODY_WIDTH):
+ result += line + "\n"
+ result += "\n"
+ newlines = 2
+ else:
+ if not onlywhite(para):
+ result += para + "\n"
+ newlines = 1
+ else:
+ if newlines < 2:
+ result += "\n"
+ newlines += 1
+ return result
+
+def hn(tag):
+ if tag[0] == 'h' and len(tag) == 2:
+ try:
+ n = int(tag[1])
+ if n in range(1, 10): return n
+ except ValueError: return 0
+
+class _html2text(HTMLParser.HTMLParser):
+ def __init__(self, out=None, baseurl=''):
+ HTMLParser.HTMLParser.__init__(self)
+
+ if out is None: self.out = self.outtextf
+ else: self.out = out
+ try:
+ self.outtext = unicode()
+ except NameError: # Python3
+ self.outtext = str()
+ self.quiet = 0
+ self.p_p = 0
+ self.outcount = 0
+ self.start = 1
+ self.space = 0
+ self.a = []
+ self.astack = []
+ self.acount = 0
+ self.list = []
+ self.blockquote = 0
+ self.pre = 0
+ self.startpre = 0
+ self.lastWasNL = 0
+ self.abbr_title = None # current abbreviation definition
+ self.abbr_data = None # last inner HTML (for abbr being defined)
+ self.abbr_list = {} # stack of abbreviations to write later
+ self.baseurl = baseurl
+
+ def outtextf(self, s):
+ self.outtext += s
+
+ def close(self):
+ HTMLParser.HTMLParser.close(self)
+
+ self.pbr()
+ self.o('', 0, 'end')
+
+ return self.outtext
+
+ def handle_charref(self, c):
+ self.o(charref(c))
+
+ def handle_entityref(self, c):
+ self.o(entityref(c))
+
+ def handle_starttag(self, tag, attrs):
+ self.handle_tag(tag, attrs, 1)
+
+ def handle_endtag(self, tag):
+ self.handle_tag(tag, None, 0)
+
+ def previousIndex(self, attrs):
+ """ returns the index of certain set of attributes (of a link) in the
+ self.a list
+
+ If the set of attributes is not found, returns None
+ """
+ if not has_key(attrs, 'href'): return None
+
+ i = -1
+ for a in self.a:
+ i += 1
+ match = 0
+
+ if has_key(a, 'href') and a['href'] == attrs['href']:
+ if has_key(a, 'title') or has_key(attrs, 'title'):
+ if (has_key(a, 'title') and has_key(attrs, 'title') and
+ a['title'] == attrs['title']):
+ match = True
+ else:
+ match = True
+
+ if match: return i
+
+ def handle_tag(self, tag, attrs, start):
+ #attrs = fixattrs(attrs)
+
+ if hn(tag):
+ self.p()
+ if start: self.o(hn(tag)*"#" + ' ')
+
+ if tag in ['p', 'div']: self.p()
+
+ if tag == "br" and start: self.o(" \n")
+
+ if tag == "hr" and start:
+ self.p()
+ self.o("* * *")
+ self.p()
+
+ if tag in ["head", "style", 'script']:
+ if start: self.quiet += 1
+ else: self.quiet -= 1
+
+ if tag in ["body"]:
+ self.quiet = 0 # sites like 9rules.com never close <head>
+
+ if tag == "blockquote":
+ if start:
+ self.p(); self.o('> ', 0, 1); self.start = 1
+ self.blockquote += 1
+ else:
+ self.blockquote -= 1
+ self.p()
+
+ if tag in ['em', 'i', 'u']: self.o("_")
+ if tag in ['strong', 'b']: self.o("**")
+ if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` ``
+ if tag == "abbr":
+ if start:
+ attrsD = {}
+ for (x, y) in attrs: attrsD[x] = y
+ attrs = attrsD
+
+ self.abbr_title = None
+ self.abbr_data = ''
+ if has_key(attrs, 'title'):
+ self.abbr_title = attrs['title']
+ else:
+ if self.abbr_title != None:
+ self.abbr_list[self.abbr_data] = self.abbr_title
+ self.abbr_title = None
+ self.abbr_data = ''
+
+ if tag == "a":
+ if start:
+ attrsD = {}
+ for (x, y) in attrs: attrsD[x] = y
+ attrs = attrsD
+ if has_key(attrs, 'href') and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')):
+ self.astack.append(attrs)
+ self.o("[")
+ else:
+ self.astack.append(None)
+ else:
+ if self.astack:
+ a = self.astack.pop()
+ if a:
+ i = self.previousIndex(a)
+ if i is not None:
+ a = self.a[i]
+ else:
+ self.acount += 1
+ a['count'] = self.acount
+ a['outcount'] = self.outcount
+ self.a.append(a)
+ self.o("][" + str(a['count']) + "]")
+
+ if tag == "img" and start:
+ attrsD = {}
+ for (x, y) in attrs: attrsD[x] = y
+ attrs = attrsD
+ if has_key(attrs, 'src'):
+ attrs['href'] = attrs['src']
+ alt = attrs.get('alt', '')
+ i = self.previousIndex(attrs)
+ if i is not None:
+ attrs = self.a[i]
+ else:
+ self.acount += 1
+ attrs['count'] = self.acount
+ attrs['outcount'] = self.outcount
+ self.a.append(attrs)
+ self.o("![")
+ self.o(alt)
+ self.o("]["+ str(attrs['count']) +"]")
+
+ if tag == 'dl' and start: self.p()
+ if tag == 'dt' and not start: self.pbr()
+ if tag == 'dd' and start: self.o(' ')
+ if tag == 'dd' and not start: self.pbr()
+
+ if tag in ["ol", "ul"]:
+ if start:
+ self.list.append({'name':tag, 'num':0})
+ else:
+ if self.list: self.list.pop()
+
+ self.p()
+
+ if tag == 'li':
+ if start:
+ self.pbr()
+ if self.list: li = self.list[-1]
+ else: li = {'name':'ul', 'num':0}
+ self.o(" "*len(self.list)) #TODO: line up <ol><li>s > 9 correctly.
+ if li['name'] == "ul": self.o("* ")
+ elif li['name'] == "ol":
+ li['num'] += 1
+ self.o(str(li['num'])+". ")
+ self.start = 1
+ else:
+ self.pbr()
+
+ if tag in ["table", "tr"] and start: self.p()
+ if tag == 'td': self.pbr()
+
+ if tag == "pre":
+ if start:
+ self.startpre = 1
+ self.pre = 1
+ else:
+ self.pre = 0
+ self.p()
+
+ def pbr(self):
+ if self.p_p == 0: self.p_p = 1
+
+ def p(self): self.p_p = 2
+
+ def o(self, data, puredata=0, force=0):
+ if self.abbr_data is not None: self.abbr_data += data
+
+ if not self.quiet:
+ if puredata and not self.pre:
+ data = re.sub('\s+', ' ', data)
+ if data and data[0] == ' ':
+ self.space = 1
+ data = data[1:]
+ if not data and not force: return
+
+ if self.startpre:
+ #self.out(" :") #TODO: not output when already one there
+ self.startpre = 0
+
+ bq = (">" * self.blockquote)
+ if not (force and data and data[0] == ">") and self.blockquote: bq += " "
+
+ if self.pre:
+ bq += " "
+ data = data.replace("\n", "\n"+bq)
+
+ if self.start:
+ self.space = 0
+ self.p_p = 0
+ self.start = 0
+
+ if force == 'end':
+ # It's the end.
+ self.p_p = 0
+ self.out("\n")
+ self.space = 0
+
+
+ if self.p_p:
+ self.out(('\n'+bq)*self.p_p)
+ self.space = 0
+
+ if self.space:
+ if not self.lastWasNL: self.out(' ')
+ self.space = 0
+
+ if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"):
+ if force == "end": self.out("\n")
+
+ newa = []
+ for link in self.a:
+ if self.outcount > link['outcount']:
+ self.out(" ["+ str(link['count']) +"]: " + urlparse.urljoin(self.baseurl, link['href']))
+ if has_key(link, 'title'): self.out(" ("+link['title']+")")
+ self.out("\n")
+ else:
+ newa.append(link)
+
+ if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done.
+
+ self.a = newa
+
+ if self.abbr_list and force == "end":
+ for abbr, definition in self.abbr_list.items():
+ self.out(" *[" + abbr + "]: " + definition + "\n")
+
+ self.p_p = 0
+ self.out(data)
+ self.lastWasNL = data and data[-1] == '\n'
+ self.outcount += 1
+
+ def handle_data(self, data):
+ if r'\/script>' in data: self.quiet -= 1
+ self.o(data, 1)
+
+ def unknown_decl(self, data): pass
+
+def wrapwrite(text):
+ text = text.encode('utf-8')
+ try: #Python3
+ sys.stdout.buffer.write(text)
+ except AttributeError:
+ sys.stdout.write(text)
+
+def html2text_file(html, out=wrapwrite, baseurl=''):
+ h = _html2text(out, baseurl)
+ h.feed(html)
+ h.feed("")
+ return h.close()
+
+def html2text(html, baseurl=''):
+ return optwrap(html2text_file(html, None, baseurl))
+
+if __name__ == "__main__":
+ baseurl = ''
+
+ p = optparse.OptionParser('%prog [(filename|url) [encoding]]',
+ version='%prog ' + __version__)
+ args = p.parse_args()[1]
+ if len(args) > 0:
+ file_ = args[0]
+ encoding = None
+ if len(args) == 2:
+ encoding = args[1]
+ if len(args) > 2:
+ p.error('Too many arguments')
+
+ if file_.startswith('http://') or file_.startswith('https://'):
+ baseurl = file_
+ j = urllib.urlopen(baseurl)
+ text = j.read()
+ if encoding is None:
+ try:
+ from feedparser import _getCharacterEncoding as enc
+ except ImportError:
+ enc = lambda x, y: ('utf-8', 1)
+ encoding = enc(j.headers, text)[0]
+ if encoding == 'us-ascii':
+ encoding = 'utf-8'
+ data = text.decode(encoding)
+
+ else:
+ data = open(file_, 'rb').read()
+ if encoding is None:
+ try:
+ from chardet import detect
+ except ImportError:
+ detect = lambda x: {'encoding': 'utf-8'}
+ encoding = detect(data)['encoding']
+ data = data.decode(encoding)
+ else:
+ data = sys.stdin.read()
+ wrapwrite(html2text(data, baseurl))
84 tornado_utils/http_test_client.py
@@ -0,0 +1,84 @@
+from urllib import urlencode
+import Cookie
+from tornado.httpclient import HTTPRequest
+from tornado import escape
+
+__version__ = '1.2'
+
+class LoginError(Exception):
+ pass
+
+class HTTPClientMixin(object):
+
+ def get(self, url, data=None, headers=None, follow_redirects=False):
+ if data is not None:
+ if isinstance(data, dict):
+ data = urlencode(data, True)
+ if '?' in url:
+ url += '&%s' % data
+ else:
+ url += '?%s' % data
+ return self._fetch(url, 'GET', headers=headers,
+ follow_redirects=follow_redirects)
+
+ def post(self, url, data, headers=None, follow_redirects=False):
+ if data is not None:
+ if isinstance(data, dict):
+ data = urlencode(data, True)
+ return self._fetch(url, 'POST', data, headers,
+ follow_redirects=follow_redirects)
+
+ def _fetch(self, url, method, data=None, headers=None, follow_redirects=True):
+ full_url = self.get_url(url)
+ request = HTTPRequest(full_url, follow_redirects=follow_redirects,
+ headers=headers, method=method, body=data)
+ self.http_client.fetch(request, self.stop)
+ return self.wait()
+
+
+class TestClient(HTTPClientMixin):
+ def __init__(self, testcase):
+ self.testcase = testcase
+ self.cookies = Cookie.SimpleCookie()
+
+ def _render_cookie_back(self):
+ return ''.join(['%s=%s;' %(x, morsel.value)
+ for (x, morsel)
+ in self.cookies.items()])
+
+ def get(self, url, data=None, headers=None, follow_redirects=False):
+ if self.cookies:
+ if headers is None:
+ headers = dict()
+ headers['Cookie'] = self._render_cookie_back()
+ response = self.testcase.get(url, data=data, headers=headers,
+ follow_redirects=follow_redirects)
+
+ self._update_cookies(response.headers)
+ return response
+
+ def post(self, url, data, headers=None, follow_redirects=False):
+ if self.cookies:
+ if headers is None:
+ headers = dict()
+ headers['Cookie'] = self._render_cookie_back()
+ response = self.testcase.post(url, data=data, headers=headers,
+ follow_redirects=follow_redirects)
+ self._update_cookies(response.headers)
+ return response
+
+ def _update_cookies(self, headers):
+ try:
+ sc = headers['Set-Cookie']
+ self.cookies.update(Cookie.SimpleCookie(
+ escape.native_str(sc)))
+ except KeyError:
+ return
+
+ def login(self, email, password, url='/auth/login/'):
+ data = dict(email=email, password=password)
+ response = self.post(url, data, follow_redirects=False)
+ if response.code != 302:
+ raise LoginError(response.body)
+ if 'Error' in response.body:
+ raise LoginError(response.body)
49 tornado_utils/routes.py
@@ -0,0 +1,49 @@
+import tornado.web
+class route(object):
+ """
+ decorates RequestHandlers and builds up a list of routables handlers
+
+ Tech Notes (or 'What the *@# is really happening here?')
+ --------------------------------------------------------
+
+ Everytime @route('...') is called, we instantiate a new route object which
+ saves off the passed in URI. Then, since it's a decorator, the function is
+ passed to the route.__call__ method as an argument. We save a reference to
+ that handler with our uri in our class level routes list then return that
+ class to be instantiated as normal.
+
+ Later, we can call the classmethod route.get_routes to return that list of
+ tuples which can be handed directly to the tornado.web.Application
+ instantiation.
+
+ Example
+ -------
+
+ @route('/some/path')
+ class SomeRequestHandler(RequestHandler):
+ pass
+
+ @route('/some/path', name='other')
+ class SomeOtherRequestHandler(RequestHandler):
+ pass
+
+ my_routes = route.get_routes()
+ """
+ _routes = []
+
+ def __init__(self, uri, name=None):
+ self._uri = uri
+ self.name = name
+
+ def __call__(self, _handler):
+ """gets called when we class decorate"""
+ name = self.name and self.name or _handler.__name__
+ self._routes.append(tornado.web.url(self._uri, _handler, name=name))
+ return _handler
+
+ @classmethod
+ def get_routes(self):
+ return self._routes
+
+def route_redirect(from_, to, name=None):
+ route._routes.append(tornado.web.url(from_, tornado.web.RedirectHandler, dict(url=to), name=name))
1 tornado_utils/send_mail/__init__.py
@@ -0,0 +1 @@
+from send_email import send_email, send_multipart_email
1 tornado_utils/send_mail/backends/__init__.py
@@ -0,0 +1 @@
+#
41 tornado_utils/send_mail/backends/base.py
@@ -0,0 +1,41 @@
+"""Base email backend class."""
+
+class BaseEmailBackend(object):
+ """
+ Base class for email backend implementations.
+
+ Subclasses must at least overwrite send_messages().
+ """
+ def __init__(self, fail_silently=False, **kwargs):
+ self.fail_silently = fail_silently
+
+ def open(self):
+ """Open a network connection.
+
+ This method can be overwritten by backend implementations to
+ open a network connection.
+
+ It's up to the backend implementation to track the status of
+ a network connection if it's needed by the backend.
+
+ This method can be called by applications to force a single
+ network connection to be used when sending mails. See the
+ send_messages() method of the SMTP backend for a reference
+ implementation.
+
+ The default implementation does nothing.
+ """
+ pass
+
+ def close(self):
+ """Close a network connection."""
+ pass
+
+ def send_messages(self, email_messages):
+ """
+ Sends one or more EmailMessage objects and returns the number of email
+ messages sent.
+ """
+ raise NotImplementedError
+
+
34 tornado_utils/send_mail/backends/console.py
@@ -0,0 +1,34 @@
+import sys
+import threading
+
+from utils.send_mail.backends.base import BaseEmailBackend
+
+class EmailBackend(BaseEmailBackend):
+ def __init__(self, *args, **kwargs):
+ self.stream = kwargs.pop('stream', sys.stdout)
+ self._lock = threading.RLock()
+ super(EmailBackend, self).__init__(*args, **kwargs)
+
+ def send_messages(self, email_messages):
+ """Write all messages to the stream in a thread-safe way."""
+ if not email_messages:
+ return
+ self._lock.acquire()
+ try:
+ # The try-except is nested to allow for
+ # Python 2.4 support (Refs #12147)
+ try:
+ stream_created = self.open()
+ for message in email_messages:
+ self.stream.write('%s\n' % message.message().as_string())
+ self.stream.write('-'*79)
+ self.stream.write('\n')
+ self.stream.flush() # flush after each message
+ if stream_created:
+ self.close()
+ except:
+ if not self.fail_silently:
+ raise
+ finally:
+ self._lock.release()
+ return len(email_messages)
26 tornado_utils/send_mail/backends/locmem.py
@@ -0,0 +1,26 @@
+"""
+Backend for test environment.
+"""
+import sys
+#print sys.path
+import utils.send_mail as mail
+#from utils.send_mail import send_email as mail
+from utils.send_mail.backends.base import BaseEmailBackend
+
+class EmailBackend(BaseEmailBackend):
+ """A email backend for use during test sessions.
+
+ The test connection stores email messages in a dummy outbox,
+ rather than sending them out on the wire.
+
+ The dummy outbox is accessible through the outbox instance attribute.
+ """
+ def __init__(self, *args, **kwargs):
+ super(EmailBackend, self).__init__(*args, **kwargs)
+ if not hasattr(mail, 'outbox'):
+ mail.outbox = []
+
+ def send_messages(self, messages):
+ """Redirect messages to the dummy outbox"""
+ mail.outbox.extend(messages)
+ return len(messages)
54 tornado_utils/send_mail/backends/pickle.py
@@ -0,0 +1,54 @@
+"""Pickling email sender"""
+
+import time
+import datetime
+import os.path
+import cPickle
+import logging
+from utils.send_mail.backends.base import BaseEmailBackend
+from utils.send_mail import config
+
+class EmailBackend(BaseEmailBackend):
+
+ def __init__(self, *args, **kwargs):
+ super(EmailBackend, self).__init__(*args, **kwargs)
+ self.location = config.PICKLE_LOCATION
+ self.protocol = getattr(config, 'PICKLE_PROTOCOL', 0)
+
+ # test that we can write to the location
+ open(os.path.join(self.location, 'test.pickle'), 'w').write('test\n')
+ os.remove(os.path.join(self.location, 'test.pickle'))
+
+ def send_messages(self, email_messages):
+ """
+ Sends one or more EmailMessage objects and returns the number of email
+ messages sent.
+ """
+ if not email_messages:
+ return
+
+ num_sent = 0
+ for message in email_messages:
+ if self._pickle(message):
+ num_sent += 1
+ return num_sent
+
+ def _pickle(self, message):
+ t0 = time.time()
+ filename = self._pickle_actual(message)
+ t1 = time.time()
+ logging.debug("Took %s seconds to create %s" % \
+ (t1 - t0, filename))
+ return True
+
+ def _pickle_actual(self, message):
+ filename_base = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+ c = 0
+ filename = os.path.join(self.location,
+ filename_base + '_%s.pickle' % c)
+ while os.path.isfile(filename):
+ c += 1
+ filename = os.path.join(self.location,
+ filename_base + '_%s.pickle' % c)
+ cPickle.dump(message, open(filename, 'wb'), self.protocol)
+ return filename
106 tornado_utils/send_mail/backends/smtp.py
@@ -0,0 +1,106 @@
+"""SMTP email backend class."""
+
+import smtplib
+import socket
+import threading
+
+from utils.send_mail.backends.base import BaseEmailBackend
+from utils.send_mail.dns_name import DNS_NAME
+from utils.send_mail import config
+
+class EmailBackend(BaseEmailBackend):
+ """
+ A wrapper that manages the SMTP network connection.
+ """
+ def __init__(self, host=None, port=None, username=None, password=None,
+ use_tls=None, fail_silently=False, **kwargs):
+ super(EmailBackend, self).__init__(fail_silently=fail_silently)
+ self.host = host or config.EMAIL_HOST
+ self.port = port or config.EMAIL_PORT
+ self.username = username or config.EMAIL_HOST_USER
+ self.password = password or config.EMAIL_HOST_PASSWORD
+ if use_tls is None:
+ self.use_tls = config.EMAIL_USE_TLS
+ else:
+ self.use_tls = use_tls
+ self.connection = None
+ self._lock = threading.RLock()
+
+ def open(self):
+ """
+ Ensures we have a connection to the email server. Returns whether or
+ not a new connection was required (True or False).
+ """
+ if self.connection:
+ # Nothing to do if the connection is already open.
+ return False
+ try:
+ # If local_hostname is not specified, socket.getfqdn() gets used.
+ # For performance, we use the cached FQDN for local_hostname.
+ self.connection = smtplib.SMTP(self.host, self.port,
+ local_hostname=DNS_NAME.get_fqdn())
+ if self.use_tls:
+ self.connection.ehlo()
+ self.connection.starttls()
+ self.connection.ehlo()
+ if self.username and self.password:
+ self.connection.login(self.username, self.password)
+ return True
+ except:
+ if not self.fail_silently:
+ raise
+
+ def close(self):
+ """Closes the connection to the email server."""
+ try:
+ try:
+ self.connection.quit()
+ except socket.sslerror:
+ # This happens when calling quit() on a TLS connection
+ # sometimes.
+ self.connection.close()
+ except:
+ if self.fail_silently:
+ return
+ raise
+ finally:
+ self.connection = None
+
+ def send_messages(self, email_messages):
+ """
+ Sends one or more EmailMessage objects and returns the number of email
+ messages sent.
+ """
+ if not email_messages:
+ return
+ self._lock.acquire()
+ try:
+ new_conn_created = self.open()
+ if not self.connection:
+ # We failed silently on open().
+ # Trying to send would be pointless.
+ return
+ num_sent = 0
+ for message in email_messages:
+ sent = self._send(message)
+ if sent:
+ num_sent += 1
+ if new_conn_created:
+ self.close()
+ finally:
+ self._lock.release()
+ return num_sent
+
+ def _send(self, email_message):
+ """A helper method that does the actual sending."""
+ if not email_message.recipients():
+ return False
+ try:
+ self.connection.sendmail(email_message.from_email,
+ email_message.recipients(),
+ email_message.message().as_string())
+ except:
+ if not self.fail_silently:
+ raise
+ return False
+ return True
13 tornado_utils/send_mail/config.py
@@ -0,0 +1,13 @@
+EMAIL_HOST = 'smtp.elasticemail.com'
+EMAIL_PORT = 2525
+
+EMAIL_HOST_USER = 'mail@peterbe.com'
+EMAIL_HOST_PASSWORD = '67255e4d-f88b-4e6a-a7d3-c8a6bfea8ea3'
+
+EMAIL_USE_TLS = False
+
+import os
+op = os.path
+PICKLE_LOCATION = op.join(op.dirname(__file__), 'pickled_messages')
+if not op.isdir(PICKLE_LOCATION):
+ os.mkdir(PICKLE_LOCATION)
30 tornado_utils/send_mail/dns_name.py
@@ -0,0 +1,30 @@
+"""
+Email message and email sending related helper functions.
+"""
+
+import socket
+
+
+# Cache the hostname, but do it lazily: socket.getfqdn() can take a couple of
+# seconds, which slows down the restart of the server.
+class CachedDnsName(object):
+ def __str__(self):
+ return self.get_fqdn()
+
+ def get_fqdn(self):
+ if not hasattr(self, '_fqdn'):
+ self._fqdn = socket.getfqdn()
+ return self._fqdn
+
+DNS_NAME = CachedDnsName()
+
+
+
+
+
+
+
+
+
+
+
36 tornado_utils/send_mail/importlib.py
@@ -0,0 +1,36 @@
+# Taken from Python 2.7 with permission from/by the original author.
+import sys
+
+def _resolve_name(name, package, level):
+ """Return the absolute name of the module to be imported."""
+ if not hasattr(package, 'rindex'):
+ raise ValueError("'package' not set to a string")
+ dot = len(package)
+ for x in xrange(level, 1, -1):
+ try:
+ dot = package.rindex('.', 0, dot)
+ except ValueError:
+ raise ValueError("attempted relative import beyond top-level "
+ "package")
+ return "%s.%s" % (package[:dot], name)
+
+
+def import_module(name, package=None):
+ """Import a module.
+
+ The 'package' argument is required when performing a relative import. It
+ specifies the package to use as the anchor point from which to resolve the
+ relative import to an absolute import.
+
+ """
+ if name.startswith('.'):
+ if not package:
+ raise TypeError("relative imports require the 'package' argument")
+ level = 0
+ for character in name:
+ if character != '.':
+ break
+ level += 1
+ name = _resolve_name(name[level:], package, level)
+ __import__(name)
+ return sys.modules[name]
57 tornado_utils/send_mail/pickled_messages/20110710_163407_0.pickle
@@ -0,0 +1,57 @@
+ccopy_reg
+_reconstructor
+p1
+(cutils.send_mail.send_email
+EmailMessage
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'body'
+p6
+S'TRACEBACK:\nTraceback (most recent call last):\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 927, in _execute\n getattr(self, self.request.method.lower())(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 327, in get\n self.render("home.html", **options)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 451, in render\n html = self.render_string(template_name, **kwargs)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 562, in render_string\n return t.generate(**args)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/template.py", line 147, in generate\n return execute()\n File "<template home.html>", line 176, in _execute\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 976, in render\n rendered = self._active_modules[name].render(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/ui_modules.py", line 30, in render\n debug=self.handler.application.settings[\'debug\']\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 1624, in render_string\n return self.handler.render_string(path, **kwargs)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 562, in render_string\n return t.generate(**args)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/template.py", line 147, in generate\n return execute()\n File "<template modules/settings.html>", line 9, in _execute\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 976, in render\n rendered = self._active_modules[name].render(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/utils/tornado_static.py", line 146, in render\n verbose=1 or self.handler.settings.get(\'debug\', False))\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/utils/tornado_static.py", line 301, in run_uglify_js_compiler\n r = _run_uglify_js_compiler(code, jar_location)\nNameError: global name \'jar_location\' is not defined\n\nREQUEST ARGUMENTS:\n{}\n\nCOOKIES:\n user: \'4db72d60df70556b0500000c\'\n\nREQUEST:\n full_url: \'http://gkc/\'\n protocol: \'http\'\n query: \'\'\n remote_ip: \'127.0.0.1\'\n request_time: 2.3128190040588379\n uri: \'/\'\n version: \'HTTP/1.0\'\n\nGIT REVISION: 2011-07-10 16:21:59\n\nHEADERS:\n{\'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',\n \'Accept-Charset\': \'ISO-8859-1,utf-8;q=0.7,*;q=0.3\',\n \'Accept-Encoding\': \'gzip,deflate,sdch\',\n \'Accept-Language\': \'en-US,en;q=0.8\',\n \'Connection\': \'close\',\n \'Cookie\': \'user=NGRiNzJkNjBkZjcwNTU2YjA1MDAwMDBj|1309622586|b2aa77afb86a85b55e63a65d79999bb1850bebc3\',\n \'Host\': \'gkc\',\n \'If-None-Match\': \'"fce9f55202560659b7c1554347f6f1892f4ebe4d"\',\n \'Referer\': \'http://gkc/play/battle\',\n \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30\',\n \'X-Scheme\': \'http\'}\n'
+p7
+sS'extra_headers'
+p8
+(dp9
+sS'attachments'
+p10
+(lp11
+sS'from_email'
+p12
+S'webmaster@kwissle.com'
+p13
+sS'to'
+p14
+(lp15
+S'peterbe@gmail.com'
+p16
+asS'connection'
+p17
+g1
+(cutils.send_mail.backends.pickle
+EmailBackend
+p18
+g3
+NtRp19
+(dp20
+S'fail_silently'
+p21
+I00
+sS'protocol'
+p22
+I0
+sS'location'
+p23
+S'/Users/peterbe/dev/TORNADO/tornado_gkc/utils/send_mail/pickled_messages'
+p24
+sbsS'bcc'
+p25
+(lp26
+sS'subject'
+p27
+S'NameError("global name \'jar_location\' is not defined",) on /'
+p28
+sb.
57 tornado_utils/send_mail/pickled_messages/20110801_112712_0.pickle
@@ -0,0 +1,57 @@
+ccopy_reg
+_reconstructor
+p1
+(cutils.send_mail.send_email
+EmailMessage
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'body'
+p6
+S'TRACEBACK:\nTraceback (most recent call last):\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 972, in _execute\n getattr(self, self.request.method.lower())(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 369, in get\n self.render("home.html", **options)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 459, in render\n html = self.render_string(template_name, **kwargs)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 570, in render_string\n return t.generate(**args)\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/template.py", line 148, in generate\n return execute()\n File "<template home.html>", line 16, in _execute\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 1023, in render\n rendered = self._active_modules[name].render(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/utils/tornado_static.py", line 267, in render\n url = super(Static, self).render(*static_urls)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/utils/tornado_static.py", line 165, in render\n os.path.dirname(old_paths[full_path])\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/utils/tornado_static.py", line 240, in _replace_css_images_with_static_urls\n css_code = _regex.sub(replacer, css_code)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/utils/tornado_static.py", line 237, in replacer\n new_filename = self.handler.static_url(os.path.join(rel_dir, filename))\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 64, in static_url\n timestamps[abs_path] = os.stat(abs_path)[stat.ST_MTIME]\nNameError: global name \'stat\' is not defined\n\nREQUEST ARGUMENTS:\n{}\n\nCOOKIES:\n __utma: None\n __utmz: None\n __utmc: None\n user: \'4d9a04d9df7055583a000000\'\n _xsrf: None\n\nREQUEST:\n full_url: \'http://gkc/\'\n protocol: \'http\'\n query: \'\'\n remote_ip: \'127.0.0.1\'\n request_time: 0.58024311065673828\n uri: \'/\'\n version: \'HTTP/1.0\'\n\nGIT REVISION: 2011-07-29 00:39:43\n\nHEADERS:\n{\'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',\n \'Accept-Charset\': \'ISO-8859-1,utf-8;q=0.7,*;q=0.7\',\n \'Accept-Encoding\': \'gzip, deflate\',\n \'Accept-Language\': \'en-us,en;q=0.5\',\n \'Connection\': \'close\',\n \'Cookie\': \'user=NGQ5YTA0ZDlkZjcwNTU1ODNhMDAwMDAw|1311198963|5e7d3e6a665b9aed013701a59086945881d9a69c; __utma=47187633.1934149131.1311670837.1311670837.1311677242.2; __utmz=47187633.1311670837.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _xsrf=6620e5a0a5c54cf5b343994f72a2bca4; __utmc=47187633\',\n \'Host\': \'gkc\',\n \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\',\n \'X-Scheme\': \'http\'}\n'
+p7
+sS'extra_headers'
+p8
+(dp9
+sS'attachments'
+p10
+(lp11
+sS'from_email'
+p12
+S'webmaster@kwissle.com'
+p13
+sS'to'
+p14
+(lp15
+S'peterbe@gmail.com'
+p16
+asS'connection'
+p17
+g1
+(cutils.send_mail.backends.pickle
+EmailBackend
+p18
+g3
+NtRp19
+(dp20
+S'fail_silently'
+p21
+I00
+sS'protocol'
+p22
+I0
+sS'location'
+p23
+S'/Users/peterbe/dev/TORNADO/tornado_gkc/utils/send_mail/pickled_messages'
+p24
+sbsS'bcc'
+p25
+(lp26
+sS'subject'
+p27
+S'NameError("global name \'stat\' is not defined",) on /'
+p28
+sb.
57 tornado_utils/send_mail/pickled_messages/20110806_150814_0.pickle
@@ -0,0 +1,57 @@
+ccopy_reg
+_reconstructor
+p1
+(cutils.send_mail.send_email
+EmailMessage
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'body'
+p6
+S'TRACEBACK:\nTraceback (most recent call last):\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 972, in _execute\n getattr(self, self.request.method.lower())(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 843, in get\n output, timestamp = _CACHED_SITEMAP_DATA\nUnboundLocalError: local variable \'_CACHED_SITEMAP_DATA\' referenced before assignment\n\nREQUEST ARGUMENTS:\n{}\n\nCOOKIES:\n _xsrf: None\n user: \'4d9a04d9df7055583a000000\'\n\nREQUEST:\n full_url: \'http://gkc/sitemap.xml\'\n protocol: \'http\'\n query: \'\'\n remote_ip: \'127.0.0.1\'\n request_time: 0.0019500255584716797\n uri: \'/sitemap.xml\'\n version: \'HTTP/1.0\'\n\nGIT REVISION: 2011-08-05 14:26:34\n\nHEADERS:\n{\'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',\n \'Accept-Charset\': \'ISO-8859-1,utf-8;q=0.7,*;q=0.7\',\n \'Accept-Encoding\': \'gzip, deflate\',\n \'Accept-Language\': \'en-us,en;q=0.5\',\n \'Cache-Control\': \'max-age=0\',\n \'Connection\': \'close\',\n \'Cookie\': \'user=NGQ5YTA0ZDlkZjcwNTU1ODNhMDAwMDAw|1312546139|f6434780157e6f3e53e71ad0f73d6797744d4921; _xsrf=b618d6941aa7409f97df1e28b216c3ff\',\n \'Host\': \'gkc\',\n \'If-None-Match\': \'"e6878db28125e9c31643727f857f5ca840da1358"\',\n \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\',\n \'X-Scheme\': \'http\'}\n'
+p7
+sS'extra_headers'
+p8
+(dp9
+sS'attachments'
+p10
+(lp11
+sS'from_email'
+p12
+S'webmaster@kwissle.com'
+p13
+sS'to'
+p14
+(lp15
+S'peterbe@gmail.com'
+p16
+asS'connection'
+p17
+g1
+(cutils.send_mail.backends.pickle
+EmailBackend
+p18
+g3
+NtRp19
+(dp20
+S'fail_silently'
+p21
+I00
+sS'protocol'
+p22
+I0
+sS'location'
+p23
+S'/Users/peterbe/dev/TORNADO/tornado_gkc/utils/send_mail/pickled_messages'
+p24
+sbsS'bcc'
+p25
+(lp26
+sS'subject'
+p27
+S'UnboundLocalError("local variable \'_CACHED_SITEMAP_DATA\' referenced before assignment",) on /sitemap.xml'
+p28
+sb.
57 tornado_utils/send_mail/pickled_messages/20110806_150826_0.pickle
@@ -0,0 +1,57 @@
+ccopy_reg
+_reconstructor
+p1
+(cutils.send_mail.send_email
+EmailMessage
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'body'
+p6
+S'TRACEBACK:\nTraceback (most recent call last):\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 972, in _execute\n getattr(self, self.request.method.lower())(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 844, in get\n output, timestamp = _CACHED_SITEMAP_DATA\nTypeError: \'NoneType\' object is not iterable\n\nREQUEST ARGUMENTS:\n{}\n\nCOOKIES:\n _xsrf: None\n user: \'4d9a04d9df7055583a000000\'\n\nREQUEST:\n full_url: \'http://gkc/sitemap.xml\'\n protocol: \'http\'\n query: \'\'\n remote_ip: \'127.0.0.1\'\n request_time: 0.0020129680633544922\n uri: \'/sitemap.xml\'\n version: \'HTTP/1.0\'\n\nGIT REVISION: 2011-08-05 14:26:34\n\nHEADERS:\n{\'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',\n \'Accept-Charset\': \'ISO-8859-1,utf-8;q=0.7,*;q=0.7\',\n \'Accept-Encoding\': \'gzip, deflate\',\n \'Accept-Language\': \'en-us,en;q=0.5\',\n \'Cache-Control\': \'max-age=0\',\n \'Connection\': \'close\',\n \'Cookie\': \'user=NGQ5YTA0ZDlkZjcwNTU1ODNhMDAwMDAw|1312546139|f6434780157e6f3e53e71ad0f73d6797744d4921; _xsrf=b618d6941aa7409f97df1e28b216c3ff\',\n \'Host\': \'gkc\',\n \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\',\n \'X-Scheme\': \'http\'}\n'
+p7
+sS'extra_headers'
+p8
+(dp9
+sS'attachments'
+p10
+(lp11
+sS'from_email'
+p12
+S'webmaster@kwissle.com'
+p13
+sS'to'
+p14
+(lp15
+S'peterbe@gmail.com'
+p16
+asS'connection'
+p17
+g1
+(cutils.send_mail.backends.pickle
+EmailBackend
+p18
+g3
+NtRp19
+(dp20
+S'fail_silently'
+p21
+I00
+sS'protocol'
+p22
+I0
+sS'location'
+p23
+S'/Users/peterbe/dev/TORNADO/tornado_gkc/utils/send_mail/pickled_messages'
+p24
+sbsS'bcc'
+p25
+(lp26
+sS'subject'
+p27
+S'TypeError("\'NoneType\' object is not iterable",) on /sitemap.xml'
+p28
+sb.
57 tornado_utils/send_mail/pickled_messages/20110806_150852_0.pickle
@@ -0,0 +1,57 @@
+ccopy_reg
+_reconstructor
+p1
+(cutils.send_mail.send_email
+EmailMessage
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'body'
+p6
+S'TRACEBACK:\nTraceback (most recent call last):\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 972, in _execute\n getattr(self, self.request.method.lower())(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 858, in get\n output = self._get_output(sites, base_url)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 872, in _get_output\n image = SubElement(url, \'image:image\')\nNameError: global name \'SubElement\' is not defined\n\nREQUEST ARGUMENTS:\n{}\n\nCOOKIES:\n _xsrf: None\n user: \'4d9a04d9df7055583a000000\'\n\nREQUEST:\n full_url: \'http://gkc/sitemap.xml\'\n protocol: \'http\'\n query: \'\'\n remote_ip: \'127.0.0.1\'\n request_time: 0.056515932083129883\n uri: \'/sitemap.xml\'\n version: \'HTTP/1.0\'\n\nGIT REVISION: 2011-08-05 14:26:34\n\nHEADERS:\n{\'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',\n \'Accept-Charset\': \'ISO-8859-1,utf-8;q=0.7,*;q=0.7\',\n \'Accept-Encoding\': \'gzip, deflate\',\n \'Accept-Language\': \'en-us,en;q=0.5\',\n \'Cache-Control\': \'max-age=0\',\n \'Connection\': \'close\',\n \'Cookie\': \'user=NGQ5YTA0ZDlkZjcwNTU1ODNhMDAwMDAw|1312546139|f6434780157e6f3e53e71ad0f73d6797744d4921; _xsrf=b618d6941aa7409f97df1e28b216c3ff\',\n \'Host\': \'gkc\',\n \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\',\n \'X-Scheme\': \'http\'}\n'
+p7
+sS'extra_headers'
+p8
+(dp9
+sS'attachments'
+p10
+(lp11
+sS'from_email'
+p12
+S'webmaster@kwissle.com'
+p13
+sS'to'
+p14
+(lp15
+S'peterbe@gmail.com'
+p16
+asS'connection'
+p17
+g1
+(cutils.send_mail.backends.pickle
+EmailBackend
+p18
+g3
+NtRp19
+(dp20
+S'fail_silently'
+p21
+I00
+sS'protocol'
+p22
+I0
+sS'location'
+p23
+S'/Users/peterbe/dev/TORNADO/tornado_gkc/utils/send_mail/pickled_messages'
+p24
+sbsS'bcc'
+p25
+(lp26
+sS'subject'
+p27
+S'NameError("global name \'SubElement\' is not defined",) on /sitemap.xml'
+p28
+sb.
57 tornado_utils/send_mail/pickled_messages/20110806_150906_0.pickle
@@ -0,0 +1,57 @@
+ccopy_reg
+_reconstructor
+p1
+(cutils.send_mail.send_email
+EmailMessage
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'body'
+p6
+S'TRACEBACK:\nTraceback (most recent call last):\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 972, in _execute\n getattr(self, self.request.method.lower())(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 858, in get\n output = self._get_output(sites, base_url)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 872, in _get_output\n image = etree.SubElement(url, \'image:image\')\nNameError: global name \'SubElement\' is not defined\n\nREQUEST ARGUMENTS:\n{}\n\nCOOKIES:\n _xsrf: None\n user: \'4d9a04d9df7055583a000000\'\n\nREQUEST:\n full_url: \'http://gkc/sitemap.xml\'\n protocol: \'http\'\n query: \'\'\n remote_ip: \'127.0.0.1\'\n request_time: 0.015753984451293945\n uri: \'/sitemap.xml\'\n version: \'HTTP/1.0\'\n\nGIT REVISION: 2011-08-05 14:26:34\n\nHEADERS:\n{\'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',\n \'Accept-Charset\': \'ISO-8859-1,utf-8;q=0.7,*;q=0.7\',\n \'Accept-Encoding\': \'gzip, deflate\',\n \'Accept-Language\': \'en-us,en;q=0.5\',\n \'Cache-Control\': \'max-age=0\',\n \'Connection\': \'close\',\n \'Cookie\': \'user=NGQ5YTA0ZDlkZjcwNTU1ODNhMDAwMDAw|1312546139|f6434780157e6f3e53e71ad0f73d6797744d4921; _xsrf=b618d6941aa7409f97df1e28b216c3ff\',\n \'Host\': \'gkc\',\n \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\',\n \'X-Scheme\': \'http\'}\n'
+p7
+sS'extra_headers'
+p8
+(dp9
+sS'attachments'
+p10
+(lp11
+sS'from_email'
+p12
+S'webmaster@kwissle.com'
+p13
+sS'to'
+p14
+(lp15
+S'peterbe@gmail.com'
+p16
+asS'connection'
+p17
+g1
+(cutils.send_mail.backends.pickle
+EmailBackend
+p18
+g3
+NtRp19
+(dp20
+S'fail_silently'
+p21
+I00
+sS'protocol'
+p22
+I0
+sS'location'
+p23
+S'/Users/peterbe/dev/TORNADO/tornado_gkc/utils/send_mail/pickled_messages'
+p24
+sbsS'bcc'
+p25
+(lp26
+sS'subject'
+p27
+S'NameError("global name \'SubElement\' is not defined",) on /sitemap.xml'
+p28
+sb.
57 tornado_utils/send_mail/pickled_messages/20110806_150910_0.pickle
@@ -0,0 +1,57 @@
+ccopy_reg
+_reconstructor
+p1
+(cutils.send_mail.send_email
+EmailMessage
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'body'
+p6
+S'TRACEBACK:\nTraceback (most recent call last):\n File "/Users/peterbe/virtualenvs/tornado_gkc/lib/python2.6/site-packages/tornado/web.py", line 972, in _execute\n getattr(self, self.request.method.lower())(*args, **kwargs)\n File "/Users/peterbe/dev/TORNADO/tornado_gkc/apps/main/handlers.py", line 859, in get\n _CACHED_SITEMAP_DATA = (output, time.time() + 60 * 60)\nAttributeError: \'builtin_function_or_method\' object has no attribute \'time\'\n\nREQUEST ARGUMENTS:\n{}\n\nCOOKIES:\n _xsrf: None\n user: \'4d9a04d9df7055583a000000\'\n\nREQUEST:\n full_url: \'http://gkc/sitemap.xml\'\n protocol: \'http\'\n query: \'\'\n remote_ip: \'127.0.0.1\'\n request_time: 0.079583883285522461\n uri: \'/sitemap.xml\'\n version: \'HTTP/1.0\'\n\nGIT REVISION: 2011-08-05 14:26:34\n\nHEADERS:\n{\'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',\n \'Accept-Charset\': \'ISO-8859-1,utf-8;q=0.7,*;q=0.7\',\n \'Accept-Encoding\': \'gzip, deflate\',\n \'Accept-Language\': \'en-us,en;q=0.5\',\n \'Cache-Control\': \'max-age=0\',\n \'Connection\': \'close\',\n \'Cookie\': \'user=NGQ5YTA0ZDlkZjcwNTU1ODNhMDAwMDAw|1312546139|f6434780157e6f3e53e71ad0f73d6797744d4921; _xsrf=b618d6941aa7409f97df1e28b216c3ff\',\n \'Host\': \'gkc\',\n \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\',\n \'X-Scheme\': \'http\'}\n'
+p7
+sS'extra_headers'
+p8
+(dp9
+sS'attachments'
+p10
+(lp11
+sS'from_email'
+p12
+S'webmaster@kwissle.com'
+p13
+sS'to'
+p14
+(lp15
+S'peterbe@gmail.com'
+p16
+asS'connection'
+p17
+g1
+(cutils.send_mail.backends.pickle
+EmailBackend
+p18
+g3
+NtRp19
+(dp20
+S'fail_silently'
+p21
+I00
+sS'protocol'
+p22
+I0
+sS'location'
+p23
+S'/Users/peterbe/dev/TORNADO/tornado_gkc/utils/send_mail/pickled_messages'
+p24
+sbsS'bcc'
+p25
+(lp26
+sS'subject'
+p27
+S'AttributeError("\'builtin_function_or_method\' object has no attribute \'time\'",) on /sitemap.xml'
+p28
+sb.
432 tornado_utils/send_mail/send_email.py
@@ -0,0 +1,432 @@
+import time
+import random
+import os
+from email.generator import Generator
+from email.Header import Header
+from email.MIMEText import MIMEText
+from email.MIMEMultipart import MIMEMultipart
+from email.Utils import formatdate
+from cStringIO import StringIO
+
+from dns_name import DNS_NAME
+from importlib import import_module
+
+class BadHeaderError(ValueError):
+ pass
+
+def force_unicode(s, encoding):
+ if isinstance(s, unicode):
+ return s
+ return unicode(s, encoding)
+
+class Promise(object):pass
+def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'):
+ """
+ Returns a bytestring version of 's', encoded as specified in 'encoding'.
+
+ If strings_only is True, don't convert (some) non-string-like objects.
+ """
+ if strings_only and isinstance(s, (types.NoneType, int)):
+ return s
+ if isinstance(s, Promise):
+ return unicode(s).encode(encoding, errors)
+ elif not isinstance(s, basestring):
+ try:
+ return str(s)
+ except UnicodeEncodeError:
+ if isinstance(s, Exception):
+ # An Exception subclass containing non-ASCII data that doesn't
+ # know how to print itself properly. We shouldn't raise a
+ # further exception.
+ return ' '.join([smart_str(arg, encoding, strings_only,
+ errors) for arg in s])
+ return unicode(s).encode(encoding, errors)
+ elif isinstance(s, unicode):
+ return s.encode(encoding, errors)
+ elif s and encoding != 'utf-8':
+ return s.decode('utf-8', errors).encode(encoding, errors)
+ else:
+ return s
+
+
+def make_msgid(idstring=None):
+ """Returns a string suitable for RFC 2822 compliant Message-ID, e.g:
+
+ <20020201195627.33539.96671@nightshade.la.mastaler.com>
+
+ Optional idstring if given is a string used to strengthen the
+ uniqueness of the message id.
+ """
+ timeval = time.time()
+ utcdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(timeval))
+ try:
+ pid = os.getpid()
+ except AttributeError:
+ # No getpid() in Jython, for example.
+ pid = 1
+ randint = random.randrange(100000)
+ if idstring is None:
+ idstring = ''
+ else:
+ idstring = '.' + idstring
+ idhost = DNS_NAME
+ msgid = '<%s.%s.%s%s@%s>' % (utcdate, pid, randint, idstring, idhost)
+ return msgid
+
+
+def forbid_multi_line_headers(name, val, encoding):
+ """Forbids multi-line headers, to prevent header injection."""
+ encoding = encoding or 'utf-8'
+ val = force_unicode(val, encoding)
+ if '\n' in val or '\r' in val:
+ raise BadHeaderError("Header values can't contain newlines (got %r for header %r)" % (val, name))
+ try:
+ val = val.encode('ascii')
+ except UnicodeEncodeError:
+ if name.lower() in ('to', 'from', 'cc'):
+ result = []
+ for nm, addr in getaddresses((val,)):
+ nm = str(Header(nm.encode(encoding), encoding))
+ result.append(formataddr((nm, str(addr))))
+ val = ', '.join(result)
+ else:
+ val = Header(val.encode(encoding), encoding)
+ else:
+ if name.lower() == 'subject':
+ val = Header(val)
+ return name, val
+
+class SafeMIMEText(MIMEText):
+
+ def __init__(self, text, subtype, charset):
+ self.encoding = charset
+ MIMEText.__init__(self, text, subtype, charset)
+
+ def __setitem__(self, name, val):
+ name, val = forbid_multi_line_headers(name, val, self.encoding)
+ MIMEText.__setitem__(self, name, val)
+
+class EmailMessage(object):
+ """
+ A container for email information.
+ """
+ content_subtype = 'plain'
+ mixed_subtype = 'mixed'
+ encoding = None # None => use settings default
+
+ def __init__(self, subject, body, from_email, to=None, bcc=None,
+ connection=None, attachments=None, headers=None):
+ """
+ Initialize a single email message (which can be sent to multiple
+ recipients).
+
+ All strings used to create the message can be unicode strings
+ (or UTF-8 bytestrings). The SafeMIMEText class will handle any
+ necessary encoding conversions.
+ """
+ if to:
+ assert not isinstance(to, basestring), '"to" argument must be a list or tuple'
+ self.to = list(to)
+ else:
+ self.to = []
+ if bcc:
+ assert not isinstance(bcc, basestring), '"bcc" argument must be a list or tuple'
+ self.bcc = list(bcc)
+ else:
+ self.bcc = []
+ self.from_email = from_email
+ self.subject = subject
+ self.body = body
+ self.attachments = attachments or []
+ self.extra_headers = headers or {}
+ self.connection = connection
+
+ def get_connection(self, fail_silently=False):
+ #from django.core.mail import get_connection
+ if not self.connection:
+ raise NotImplementedError
+ #self.connection = get_connection(fail_silently=fail_silently)
+ return self.connection
+
+ def message(self):
+ encoding = self.encoding or 'utf-8'
+
+ msg = SafeMIMEText(smart_str(self.body, encoding),
+ self.content_subtype, encoding)
+ msg = self._create_message(msg)
+ msg['Subject'] = self.subject
+ msg['From'] = self.extra_headers.get('From', self.from_email)
+ msg['To'] = ', '.join(self.to)
+
+ # Email header names are case-insensitive (RFC 2045), so we have to
+ # accommodate that when doing comparisons.
+ header_names = [key.lower() for key in self.extra_headers]
+ if 'date' not in header_names:
+ msg['Date'] = formatdate()
+ if 'message-id' not in header_names:
+ msg['Message-ID'] = make_msgid()
+ for name, value in self.extra_headers.items():
+ if name.lower() == 'from': # From is already handled
+ continue
+ msg[name] = value
+ return msg
+
+ def recipients(self):
+ """
+ Returns a list of all recipients of the email (includes direct
+ addressees as well as Bcc entries).
+ """
+ return self.to + self.bcc
+
+ def send(self, fail_silently=False):
+ """Sends the email message."""
+ if not self.recipients():
+ # Don't bother creating the network connection if there's nobody to
+ # send to.
+ return 0
+ return self.get_connection(fail_silently).send_messages([self])
+
+ def attach(self, filename=None, content=None, mimetype=None):
+ """
+ Attaches a file with the given filename and content. The filename can
+ be omitted and the mimetype is guessed, if not provided.
+
+ If the first parameter is a MIMEBase subclass it is inserted directly
+ into the resulting message attachments.
+ """
+ if isinstance(filename, MIMEBase):
+ assert content == mimetype == None
+ self.attachments.append(filename)
+ else:
+ assert content is not None
+ self.attachments.append((filename, content, mimetype))
+
+ def attach_file(self, path, mimetype=None):
+ """Attaches a file from the filesystem."""
+ filename = os.path.basename(path)
+ content = open(path, 'rb').read()
+ self.attach(filename, content, mimetype)
+
+ def _create_message(self, msg):
+ return self._create_attachments(msg)
+
+ def _create_attachments(self, msg):
+ if self.attachments:
+ encoding = self.encoding or settings.DEFAULT_CHARSET
+ body_msg = msg
+ msg = SafeMIMEMultipart(_subtype=self.mixed_subtype, encoding=encoding)
+ if self.body:
+ msg.attach(body_msg)
+ for attachment in self.attachments:
+ if isinstance(attachment, MIMEBase):
+ msg.attach(attachment)
+ else:
+ msg.attach(self._create_attachment(*attachment))
+ return msg
+
+ def _create_mime_attachment(self, content, mimetype):
+ """
+ Converts the content, mimetype pair into a MIME attachment object.
+ """
+ basetype, subtype = mimetype.split('/', 1)
+ if basetype == 'text':
+ encoding = self.encoding or 'utf-8'
+ attachment = SafeMIMEText(smart_str(content, encoding), subtype, encoding)
+ else:
+ # Encode non-text attachments with base64.
+ attachment = MIMEBase(basetype, subtype)
+ attachment.set_payload(content)
+ Encoders.encode_base64(attachment)
+ return attachment
+
+ def _create_attachment(self, filename, content, mimetype=None):
+ """
+ Converts the filename, content, mimetype triple into a MIME attachment
+ object.
+ """
+ if mimetype is None:
+ mimetype, _ = mimetypes.guess_type(filename)
+ if mimetype is None:
+ mimetype = DEFAULT_ATTACHMENT_MIME_TYPE
+ attachment = self._create_mime_attachment(content, mimetype)
+ if filename:
+ attachment.add_header('Content-Disposition', 'attachment',
+ filename=filename)
+ return attachment
+
+class EmailMultiAlternatives(EmailMessage):
+ """
+ A version of EmailMessage that makes it easy to send multipart/alternative
+ messages. For example, including text and HTML versions of the text is
+ made easier.
+ """
+ alternative_subtype = 'alternative'
+
+ def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
+ connection=None, attachments=None, headers=None, alternatives=None,
+ cc=None):
+ """
+ Initialize a single email message (which can be sent to multiple
+ recipients).
+
+ All strings used to create the message can be unicode strings (or UTF-8
+ bytestrings). The SafeMIMEText class will handle any necessary encoding
+ conversions.
+ """
+ if cc:
+ raise NotImplementedError
+ super(EmailMultiAlternatives, self).__init__(
+ subject, body, from_email, to,
+ bcc=bcc,
+ connection=connection,
+ attachments=attachments,
+ headers=headers,
+ )
+ self.alternatives = alternatives or []
+
+ def attach_alternative(self, content, mimetype):
+ """Attach an alternative content representation."""
+ assert content is not None
+ assert mimetype is not None
+ self.alternatives.append((content, mimetype))
+
+ def _create_message(self, msg):
+ return self._create_attachments(self._create_alternatives(msg))
+
+ def _create_alternatives(self, msg):
+ encoding = self.encoding or 'utf-8'
+ if self.alternatives:
+ body_msg = msg
+ msg = SafeMIMEMultipart(_subtype=self.alternative_subtype, encoding=encoding)
+ if self.body:
+ msg.attach(body_msg)
+ for alternative in self.alternatives:
+ msg.attach(self._create_mime_attachment(*alternative))
+ return msg
+
+
+class SafeMIMEText(MIMEText):
+
+ def __init__(self, text, subtype, charset):
+ self.encoding = charset
+ MIMEText.__init__(self, text, subtype, charset)
+
+ def __setitem__(self, name, val):
+ name, val = forbid_multi_line_headers(name, val, self.encoding)
+ MIMEText.__setitem__(self, name, val)
+
+ def as_string(self, unixfrom=False):
+ """Return the entire formatted message as a string.
+ Optional `unixfrom' when True, means include the Unix From_ envelope
+ header.
+
+ This overrides the default as_string() implementation to not mangle
+ lines that begin with 'From '. See bug #13433 for details.
+ """
+ fp = StringIO()
+ g = Generator(fp, mangle_from_ = False)
+ g.flatten(self, unixfrom=unixfrom)
+ return fp.getvalue()
+
+
+class SafeMIMEMultipart(MIMEMultipart):
+
+ def __init__(self, _subtype='mixed', boundary=None, _subparts=None, encoding=None, **_params):
+ self.encoding = encoding
+ MIMEMultipart.__init__(self, _subtype, boundary, _subparts, **_params)
+
+ def __setitem__(self, name, val):
+ name, val = forbid_multi_line_headers(name, val, self.encoding)
+ MIMEMultipart.__setitem__(self, name, val)
+
+ def as_string(self, unixfrom=False):
+ """Return the entire formatted message as a string.
+ Optional `unixfrom' when True, means include the Unix From_ envelope
+ header.
+
+ This overrides the default as_string() implementation to not mangle
+ lines that begin with 'From '. See bug #13433 for details.
+ """
+ fp = StringIO()
+ g = Generator(fp, mangle_from_ = False)
+ g.flatten(self, unixfrom=unixfrom)
+ return fp.getvalue()
+
+
+def get_connection(backend, fail_silently=False, **kwds):
+ """Load an e-mail backend and return an instance of it.
+
+ Both fail_silently and other keyword arguments are used in the
+ constructor of the backend.
+ """
+ path = backend# or settings.EMAIL_BACKEND
+ try:
+ mod_name, klass_name = path.rsplit('.', 1)
+ mod = import_module(mod_name)
+ except ImportError, e:
+ raise
+ klass = getattr(mod, klass_name)
+ return klass(fail_silently=fail_silently, **kwds)
+
+def send_email(backend, subject, message, from_email, recipient_list,
+ fail_silently=False, auth_user=None, auth_password=None,
+ connection=None, headers=None):
+ """
+ Easy wrapper for sending a single message to a recipient list. All members
+ of the recipient list will see the other recipients in the 'To' field.
+
+ If auth_user is None, the EMAIL_HOST_USER setting is used.
+ If auth_password is None, the EMAIL_HOST_PASSWORD setting is used.
+
+ Note: The API for this method is frozen. New code wanting to extend the
+ functionality should use the EmailMessage class directly.
+ """
+ connection = connection or get_connection(backend,
+ username=auth_user,
+ password=auth_password,
+ fail_silently=fail_silently)
+ return EmailMessage(subject, message, from_email, recipient_list,
+ connection=connection,
+ headers=headers).send()
+
+def send_multipart_email(backend,
+ text_part, html_part, subject, recipients,
+ sender, fail_silently=False, bcc=None,
+ auth_user=None, auth_password=None,
+ connection=None):
+ """
+ This function will send a multi-part e-mail with both HTML and
+ Text parts.
+
+ template_name must NOT contain an extension. Both HTML (.html) and TEXT
+ (.txt) versions must exist, eg 'emails/public_submit' will use both
+ public_submit.html and public_submit.txt.
+
+ email_context should be a plain python dictionary. It is applied against
+ both the email messages (templates) & the subject.
+
+ subject can be plain text or a Django template string, eg:
+ New Job: {{ job.id }} {{ job.title }}
+
+ recipients can be either a string, eg 'a@b.com' or a list, eg:
+ ['a@b.com', 'c@d.com']. Type conversion is done if needed.
+
+ sender can be an e-mail, 'Name <email>' or None. If unspecified, the
+ DEFAULT_FROM_EMAIL will be used.
+
+ """
+
+ if not isinstance(recipients, list):
+ recipients = [recipients]
+ if bcc is not None and not isinstance(bcc, list):
+ bcc = [bcc]
+
+ connection = connection or get_connection(backend,
+ username=auth_user,
+ password=auth_password,
+ fail_silently=fail_silently)
+ msg = EmailMultiAlternatives(subject, text_part, sender, recipients,
+ connection=connection,
+ bcc=bcc)
+ msg.attach_alternative(html_part, "text/html")
+ return msg.send(fail_silently)
42 tornado_utils/stopwords.py
@@ -0,0 +1,42 @@
+# Performance note: I benchmarked this code using a set instead of
+# a list for the stopwords and was surprised to find that the list
+# performed /better/ than the set - maybe because it's only a small
+# list.
+
+stopwords = '''
+i
+a
+an
+are
+as
+at
+be
+by
+for
+from
+how
+in
+is
+it
+of
+on
+or
+that
+the
+this
+to
+was
+what
+when
+where
+'''.split()
+
+def strip_stopwords(sentence):
+ "Removes stopwords - also normalizes whitespace"
+ words = sentence.split()
+ sentence = []
+ for word in words:
+ if word.lower() not in stopwords:
+ sentence.append(word)
+ return u' '.join(sentence)
+
129 tornado_utils/thumbnailer.py
@@ -0,0 +1,129 @@
+try:
+ from PIL import Image
+except ImportError:
+ Image = None
+import os
+
+def _mkdir(newdir):
+ """works the way a good mkdir should :)
+ - already exists, silently complete
+ - regular file in the way, raise an exception
+ - parent directory(ies) does not exist, make them as well
+ """
+ if os.path.isdir(newdir):
+ pass
+ elif os.path.isfile(newdir):
+ raise OSError("a file with the same name as the desired " \
+ "dir, '%s', already exists." % newdir)
+ else:
+ head, tail = os.path.split(newdir)
+ if head and not os.path.isdir(head):
+ _mkdir(head)
+ if tail:
+ os.mkdir(newdir)
+
+def get_thumbnail(save_path, image_data, (max_width, max_height), quality=85):
+ if not Image:
+ raise SystemError("PIL.Image was not imported")
+
+ if os.path.isfile(save_path):
+ image = Image.open(save_path)
+ #print "FOUND", save_path
+ return image.size
+ directory = os.path.dirname(save_path)
+ _mkdir(directory)
+ basename = os.path.basename(save_path)
+ original_save_path = os.path.join(directory, 'original.' + basename)
+ with open(original_save_path, 'wb') as f:
+ f.write(image_data)
+ #print "WROTE", original_save_path
+ original_image = Image.open(original_save_path)
+ image = scale_and_crop(original_image, (max_width, max_height))
+ format = None
+ try:
+ image.save(save_path,
+ format=format,
+ quality=quality,
+ optimize=1)
+ #print "SAVED", save_path
+
+ except IOError:
+ # Try again, without optimization (PIL can't optimize an image
+ # larger than ImageFile.MAXBLOCK, which is 64k by default)
+ image.save(save_path,
+ format=format,
+ quality=quality)
+
+ os.remove(original_save_path)
+ return image.size
+
+
+def scale_and_crop(im, requested_size, **opts):
+ x, y = [float(v) for v in im.size]
+ xr, yr = [float(v) for v in requested_size]
+
+ if 'crop' in opts or 'max' in opts:
+ r = max(xr / x, yr / y)
+ else:
+ r = min(xr / x, yr / y)
+
+ if r < 1.0 or (r > 1.0 and 'upscale' in opts):
+ im = im.resize((int(round(x * r)), int(round(y * r))),
+ resample=Image.ANTIALIAS)
+
+ crop = opts.get('crop') or 'crop' in opts
+ if crop:
+ # Difference (for x and y) between new image size and requested size.
+ x, y = [float(v) for v in im.size]
+ dx, dy = (x - min(x, xr)), (y - min(y, yr))
+ if dx or dy:
+ # Center cropping (default).
+ ex, ey = dx / 2, dy / 2
+ box = [ex, ey, x - ex, y - ey]
+ # See if an edge cropping argument was provided.
+ edge_crop = (isinstance(crop, basestring) and
+ re.match(r'(?:(-?)(\d+))?,(?:(-?)(\d+))?$', crop))
+ if edge_crop and filter(None, edge_crop.groups()):
+ x_right, x_crop, y_bottom, y_crop = edge_crop.groups()
+ if x_crop:
+ offset = min(x * int(x_crop) / 100, dx)
+ if x_right:
+ box[0] = dx - offset
+ box[2] = x - offset
+ else:
+ box[0] = offset
+ box[2] = x - (dx - offset)
+ if y_crop:
+ offset = min(y * int(y_crop) / 100, dy)
+ if y_bottom:
+ box[1] = dy - offset
+ box[3] = y - offset
+ else:
+ box[1] = offset
+ box[3] = y - (dy - offset)
+ # See if the image should be "smart cropped".
+ elif crop == 'smart':
+ left = top = 0
+ right, bottom = x, y
+ while dx:
+ slice = min(dx, 10)
+ l_sl = im.crop((0, 0, slice, y))
+ r_sl = im.crop((x - slice, 0, x, y))
+ if utils.image_entropy(l_sl) >= utils.image_entropy(r_sl):
+ right -= slice
+ else:
+ left += slice
+ dx -= slice
+ while dy:
+ slice = min(dy, 10)
+ t_sl = im.crop((0, 0, x, slice))
+ b_sl = im.crop((0, y - slice, x, y))
+ if utils.image_entropy(t_sl) >= utils.image_entropy(b_sl):
+ bottom -= slice
+ else:
+ top += slice
+ dy -= slice
+ box = (left, top, right, bottom)
+ # Finally, crop the image!
+ im = im.crop([int(round(v)) for v in box])
+ return im
134 tornado_utils/timesince.py
@@ -0,0 +1,134 @@
+import datetime
+
+# Language constants
+MINUTE = 'minute'
+MINUTES = 'minutes'
+HOUR = 'hour'
+HOURS = 'hours'
+YEAR = 'year'
+YEARS = 'years'
+MONTH = 'month'
+MONTHS = 'months'
+WEEK = 'week'
+WEEKS = 'weeks'
+DAY = 'day'
+DAYS = 'days'
+AND = 'and'
+
+
+#@register.filter
+def smartertimesince(d, now=None):
+ if not isinstance(d, datetime.datetime):
+ d = datetime.datetime(d.year, d.month, d.day)
+ if now and not isinstance(now, datetime.datetime):
+ now = datetime.datetime(now.year, now.month, now.day)
+
+ if not now:
+ if d.tzinfo:
+ raise NotImplementedError
+ from django.utils.tzinfo import LocalTimezone
+ now = datetime.datetime.now(LocalTimezone(d))
+ else:
+ now = datetime.datetime.now()
+
+ r = timeSince(d, now, max_no_sections=1, minute_granularity=True)
+ if not r:
+ return "seconds"
+ return r
+
+# Copied and adopted from FriedZopeBase
+def timeSince(firstdate, seconddate, afterword=None,
+ minute_granularity=False,
+ max_no_sections=3):
+ """
+ Use two date objects to return in plain english the difference between them.
+ E.g. "3 years and 2 days"
+ or "1 year and 3 months and 1 day"
+
+ Try to use weeks when the no. of days > 7
+
+ If less than 1 day, return number of hours.
+
+ If there is "no difference" between them, return false.
+ """
+
+ def wrap_afterword(result, afterword=afterword):
+ if afterword is not None:
+ return "%s %s" % (result, afterword)
+ else:
+ return result
+
+ fdo = firstdate
+ sdo = seconddate
+
+ day_difference = abs(sdo-fdo).days
+
+ years = day_difference/365
+ months = (day_difference % 365)/30
+ days = (day_difference % 365) % 30
+ minutes = ((day_difference % 365) % 30) % 24
+
+
+ if days == 0 and months == 0 and years == 0:
+ # use hours
+ hours = abs(sdo-fdo).seconds/3600
+ if hours == 1:
+ return wrap_afterword("1 %s" % (HOUR))
+ elif hours > 0: