Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Branch: master
Fetching contributors…

Cannot retrieve contributors at this time

1614 lines (1316 sloc) 47.178 kB
# -*- coding: iso-8859-1 -*-
##
## Fry-IT Zope Base utility scripts
## Peter Bengtsson, peter@fry-it.com, (c) 2005
##
# python
import os
import string
import sys
import re
import inspect
import random
from time import strftime
from random import shuffle
from math import floor
import httplib, urllib
from urlparse import urlparse, urlunparse
from types import FloatType, IntType
from urllib import urlencode
import itertools
import unicodedata
from datetime import date
from htmlentitydefs import entitydefs
entitydefs_inverted = {}
for k,v in entitydefs.items():
entitydefs_inverted[v] = k
from addhrefs import addhrefs
from unaccent import unaccented_map
# zope
try:
from DocumentTemplate import sequence
from Products.PythonScripts.standard import html_quote, newline_to_br, \
structured_text, url_quote, url_quote_plus, thousands_commas
from DateTime import DateTime
from Acquisition import aq_base
except ImportError:
pass
#from DocumentTemplate import sequence
#from Products.PythonScripts.standard import html_quote, newline_to_br, \
# structured_text, url_quote, url_quote_plus, thousands_commas
#-----------------------------------------------------------------------------
#
# ZODB
#
## https://bugs.launchpad.net/zope2/+bug/142399
def safe_hasattr(obj, name, _marker=object()):
"""Make sure we don't mask exceptions like hasattr().
We don't want exceptions other than AttributeError to be masked,
since that too often masks other programming errors.
Three-argument getattr() doesn't mask those, so we use that to
implement our own hasattr() replacement.
"""
return getattr(obj, name, _marker) is not _marker
def base_hasattr(obj, name):
"""Like safe_hasattr, but also disables acquisition."""
return safe_hasattr(aq_base(obj), name)
#-----------------------------------------------------------------------------
#
# Debuggning
#
def debug(s, tabs=0, steps=(1,), include_module_name=False):
inspect_dbg = []
if isinstance(steps, int):
steps = range(1, steps+1)
for i in steps:
try:
caller_module = inspect.stack()[i][1]
caller_method = inspect.stack()[i][3]
caller_method_line = inspect.stack()[i][2]
except IndexError:
break
if include_module_name:
if caller_module.endswith('/__init__.py'):
caller_module = '/'.join(caller_module.split('/')[-2:])
else:
caller_module = caller_module.split('/')[-1]
inspect_dbg.append("(%s)%s:%s"%(caller_module, caller_method, caller_method_line))
else:
inspect_dbg.append("%s:%s"%(caller_method, caller_method_line))
inspect_appendix = ", ".join(inspect_dbg)
if inspect_appendix:
out = "\t"*tabs + "%s (%s)"%(s, inspect_appendix)
else:
out = "\t"*tabs + "%s"%s
print out
#-----------------------------------------------------------------------------
#
# Unicode
#
def unaccent_string(ustring, encoding="ascii"):
if not isinstance(ustring, unicode):
ustring = ustring.decode(encoding)
return ustring.translate(unaccented_map()).encode(encoding, "ignore")
def unicodify(i, encoding='latin-1'):
if type(i) == unicode:
return i
else:
return unicode(str(i), encoding)
def safe_unicodify(s, encodings=('utf8','latin1')):
if isinstance(s, str):
if not isinstance(encodings, (tuple, list)):
encodings = [encodings]
for encoding in encodings:
try:
return unicode(s, encoding)
except UnicodeDecodeError:
pass
raise UnicodeDecodeError, \
"Unable to unicodify %r with these encodings %s" % (s, encodings)
return s
def internationalizeID(y, encoding='latin-1'):
""" this is an improvement to the previous internationalizeID() function
which is left above.
Taken from the comment by Aaron Bentley on
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/251871
"""
if isinstance(y, str):
y = unicode(y, encoding)
return unicodedata.normalize('NFKD', y).encode('ascii','ignore')
from types import StringType, UnicodeType, InstanceType
nasty_exception_str = Exception.__str__.im_func
def ustr(v):
"""Convert any object to a plain string or unicode string,
minimising the chance of raising a UnicodeError. This
even works with uncooperative objects like Exceptions
"""
string_types = (StringType,UnicodeType)
if type(v) in string_types:
return v
else:
fn = getattr(v,'__str__',None)
if fn is not None:
# An object that wants to present its own string representation,
# but we dont know what type of string. We cant use any built-in
# function like str() or unicode() to retrieve it because
# they all constrain the type which potentially raises an exception.
# To avoid exceptions we have to call __str__ direct.
if getattr(fn,'im_func',None)==nasty_exception_str:
# Exception objects have been optimised into C, and their
# __str__ function fails when given a unicode object.
# Unfortunately this scenario is all too common when
# migrating to unicode, because of code which does:
# raise ValueError(something_I_wasnt_expecting_to_be_unicode)
return _exception_str(v)
else:
# Trust the object to do this right
v = fn()
if type(v) in string_types:
return v
else:
raise ValueError('__str__ returned wrong type')
# Drop through for non-instance types, and instances that
# do not define a special __str__
return str(v)
def _exception_str(exc):
if hasattr(exc, 'args'):
if not exc.args:
return ''
elif len(exc.args) == 1:
return ustr(exc.args[0])
else:
return str(exc.args)
return str(exc)
#-----------------------------------------------------------------------------
#
# Internet, sockets, networking
#
def anyPost(url, cgi_params={}):
params = urllib.urlencode(cgi_params)
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
url_parsed = urlparse(url)
conn = httplib.HTTPConnection(url_parsed[1])
conn.request("POST", url_parsed[2], params, headers)
response = conn.getresponse()
res = response.status, response.reason, response.read()
conn.close()
return res
#-----------------------------------------------------------------------------
#
# URL handling
#
def parametrize_url(url, unicode_encoding='utf8', doseq=False, **params):
""" don't just add the **params because the url
itself might contain CGI variables embedded inside
the string. """
url_parsed = list(urlparse(url))
for k, v in params.items():
if isinstance(v, unicode):
params[k] = v.encode(unicode_encoding)
encoded = urlencode(params, doseq=doseq)
qs = url_parsed[4]
if encoded:
if qs:
qs += '&'+encoded
else:
qs = encoded
netloc = url_parsed[1]
if netloc.find('?')>-1:
url_parsed[1] = url_parsed[1][:netloc.find('?')]
if qs:
qs = netloc[netloc.find('?')+1:]+'&'+qs
else:
qs = netloc[netloc.find('?')+1:]
url_parsed[4] = qs
if url_parsed[2] == '':
url_parsed[2] = '/'
url = urlunparse(url_parsed)
return url
def AddParam2URL(url, params={}):
""" return url and append params but be aware of existing params """
return parametrize_url(url, **params)
p='?'
if p in url:
p = '&'
url = url + p
for key, value in params.items():
url = url + '%s=%s&'%(key, url_quote(value))
return url[:-1]
#-----------------------------------------------------------------------------
#
# Zope inspired
#
def cookIdAndTitle(s):
""" if s='image1~Image One.gif' then return ['image1.gif','Image One']
Testwork:
s='image1~Image One.gif' => ['image1.gif','Image One']
s='image1.gif' => ['image1.gif','']
s='image1~.gif' => ['image1.gif','']
s='image1~Image One.gif.gif'=> ['image1.gif','Image One.gif']
"""
if s.find('~') == -1:
return [s, '']
splitted = s.split('~',1)
id = splitted[0]
rest = s.replace(id+'~','')
if rest.rfind('.') == -1:
return [id, rest]
else:
pos_last_dot = rest.rfind('.')
ext = rest[pos_last_dot:]
id = id.strip() + ext.strip()
title = rest[0:pos_last_dot].strip()
return [id, title]
hide_key={'HTTP_AUTHORIZATION':1,
'HTTP_CGI_AUTHORIZATION': 1,
}.has_key
def REQUEST2String(REQUEST):
result="form\n"
row = "\t%s: %s\n"
for k,v in _filterPasswordFields(REQUEST.form.items()):
result=result + row % (escape(k), escape(repr(v)))
result += "cookies\n"
for k,v in _filterPasswordFields(REQUEST.cookies.items()):
result=result + row % (escape(k), escape(repr(v)))
result += "lazy items\n"
for k,v in _filterPasswordFields(REQUEST._lazies.items()):
result=result + row % (escape(k), escape(repr(v)))
result += "other\n"
for k,v in _filterPasswordFields(REQUEST.other.items()):
if k in ('PARENTS','RESPONSE'): continue
result=result + row % (escape(k), escape(repr(v)))
for n in "0123456789":
key = "URL%s"%n
try: result=result + row % (key, escape(REQUEST[key]))
except KeyError: pass
for n in "0123456789":
key = "BASE%s"%n
try: result=result + row % (key, escape(REQUEST[key]))
except KeyError: pass
result=result+"environ\n"
for k,v in REQUEST.environ.items():
if not hide_key(k):
result=result + row % (escape(k), escape(repr(v)))
return result
#-----------------------------------------------------------------------------
#
# Quoting
#
def tex_quote(s):
for k, v in {'&':r'\&', '$':r'\$', '_':r'\_', r'%':r'\%',
'|':r'\textbackslash', '#':r'\#', '^':r'\^{}',
'~':r'\~{}',
'<':'\textless ', # notice the extra space
'>':'\textgreater ', # notice the extra space
}.items():
s = s.replace(k, v)
return s
def tag_quote(text):
""" similiar to html_quote but only fix < and > """
return text.replace('<','&lt;').replace('>','&gt;')
destroyed_hex_entities = re.compile('&amp;#(\d+);')
def safe_html_quote(text):
""" like html_quote but allow things like &#1234; """
text = html_quote(text)
text = destroyed_hex_entities.sub(r'&#\1;', text)
return text
_badchars_regex = re.compile('|'.join(entitydefs.values()))
_been_fixed_regex = re.compile('&\w+;|&#[0-9]+;')
def html_entity_fixer(text, skipchars=[], extra_careful=1):
if not text:
# then don't even begin to try to do anything
return text
if isinstance(text, unicode):
return text.encode('ascii', 'xmlcharrefreplace')
# if extra_careful we don't attempt to do anything to
# the string if it might have been converted already.
if extra_careful and _been_fixed_regex.findall(text):
return text
if isinstance(skipchars, basestring):
skipchars = [skipchars]
keyholder= {}
for x in _badchars_regex.findall(text):
if x not in skipchars:
keyholder[x] = 1
text = text.replace('&','&amp;')
text = text.replace('\x80', '&#8364;')
for each in keyholder.keys():
if each == '&':
continue
better = entitydefs_inverted[each]
if not better.startswith('&#'):
better = '&%s;'%entitydefs_inverted[each]
text = text.replace(each, better)
return text
#-----------------------------------------------------------------------------
#
# Randomness
#
def nicepass(alpha=6,numeric=2):
"""
returns a human-readble password (say rol86din instead of
a difficult to remember K8Yn9muL )
"""
vowels = ['a','e','i','o','u']
consonants = [a for a in string.ascii_lowercase if a not in vowels]
digits = string.digits
####utility functions
def a_part(slen):
ret = ''
for i in range(slen):
if i%2 ==0:
randid = random.randint(0,20) #number of consonants
ret += consonants[randid]
else:
randid = random.randint(0,4) #number of vowels
ret += vowels[randid]
return ret
def n_part(slen):
ret = ''
for i in range(slen):
randid = random.randint(0,9) #number of digits
ret += digits[randid]
return ret
####
fpl = alpha/2
if alpha % 2 :
fpl = int(alpha/2) + 1
lpl = alpha - fpl
start = a_part(fpl)
mid = n_part(numeric)
end = a_part(lpl)
return "%s%s%s" % (start,mid,end)
def getRandomString(length=10, loweronly=1, numbersonly=0,
more_numbers=0, avoidchars=[], lettersonly=0):
""" return a very random string """
if numbersonly:
l = list('0123456789')
else:
if lettersonly:
lowercase = 'abcdefghijklmnopqrstuvwxyz'
else:
lowercase = 'abcdefghijklmnopqrstuvwxyz'+'0123456789'
if more_numbers and abs(more_numbers)==more_numbers:
for i in range(more_numbers):
lowercase += '0123456789'
if loweronly:
l = list(lowercase)
else:
l = list(lowercase + lowercase.upper())
if avoidchars:
if type(avoidchars)==type('s'):
avoidchars = list(avoidchars)
for e in avoidchars:
while e in l:
l.remove(e)
shuffle(l)
s = string.join(l,'')
if len(s) < length:
s = s + getRandomString(loweronly=1)
s = s[:length]
if loweronly:
return s.lower()
else:
return s
#-----------------------------------------------------------------------------
#
# Sequences, lists
#
def anyTrue(pred, seq):
""" http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/212959 """
return True in itertools.imap(pred,seq)
def anyFalse(pred, seq):
""" http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/212959 """
return False in itertools.imap(pred,seq)
def insensitiveRemove(item, list):
rm_these = []
for each in list:
if ss(each) == ss(item):
rm_these.append(each)
for each in rm_these:
list.remove(each)
def insensitiveIN(item, list):
if same_type(item, 's'):
item = ss(item)
for each in list:
if same_type(each, 's'):
if item == ss(each):
return 1
return 0
def listintersection(list1, list2):
checked = []
for e in list1:
if e in list2:
checked.append(e)
return checked
def mergeEnglishList(seq, lastword='or'):
""" make ['abc','def','efg'] into 'abc, def or efg' """
if len(seq) == 0:
return ''
elif len(seq) == 1:
return seq[0]
else:
s = ''
for i in range(len(seq)):
if i == len(seq)-2:
t = ' %s '%lastword
elif i == len(seq)-1:
t = ''
else:
t = ', '
s += "%s%s"%(seq[i], t)
return s
def uniqify(seq, idfun=None): # Alex Martelli ******* order preserving
if idfun is None:
def idfun(x): return x
seen = {}
result = []
for item in seq:
marker = idfun(item)
# in old Python versions:
# if seen.has_key(marker)
# but in new ones:
##if marker in seen: continue
if seen.has_key(marker): continue
seen[marker] = 1
result.append(item)
return result
def uniqify(seq, idfun=None): # Alex Martelli ******* order preserving
if idfun is None:
def idfun(x): return x
seen = {}
result = []
for item in seq:
marker = idfun(item)
# in old Python versions:
# if seen.has_key(marker)
# but in new ones:
##if marker in seen: continue
if seen.has_key(marker): continue
seen[marker] = 1
result.append(item)
return result
def iuniqify(seq):
""" return a list of strings unique case insensitively.
If the input is ['foo','bar','Foo']
return ['foo','bar']
"""
def idfunction(x):
if isinstance(x, basestring):
return x.lower()
else:
return x
return uniqify(seq, idfunction)
def moveUpListelement(element, xlist):
""" move an element in a _mutable_ list up one position
if possible. If the element is a list, then the function
is self recursivly called for each subelement.
"""
assert type(xlist)==type([]), "List to change not of list type "\
"(%r)"%type(xlist)
if type(element)==type([]):
for subelement in element:
moveUpListelement(subelement, xlist)
if element==xlist[0]:
pass
elif element in xlist:
i=xlist.index(element)
xlist[i], xlist[i-1] = xlist[i-1], xlist[i]
#-----------------------------------------------------------------------------
#
# HTML handling
#
starttag_regex = re.compile('<.*?>', re.MULTILINE|re.DOTALL)
singletontag_regex = re.compile('<.*?/>', re.MULTILINE|re.DOTALL)
endtag_regex = re.compile('</.*?>', re.MULTILINE|re.DOTALL)
multiple_linebreaks = re.compile('\n\s+\n|\n\n')
def dehtmlify(html):
""" Try to convert an HTML chunk to text """
li_tags = re.compile('<li>\s*',re.I)
html = li_tags.sub(' * ', html)
html = multiple_linebreaks.sub('\n', html)
html = starttag_regex.sub('',html)
html = singletontag_regex.sub('', html)
html = endtag_regex.sub('\n', html)
return html.strip()
#-----------------------------------------------------------------------------
#
# String enhancing
#
def ordinalth(n, html=0):
t = 'th st nd rd th th th th th th'.split()
if n % 100 in (11, 12, 13): #special case
if html:
return '%d'%n+'<sup>th</sup>'
else:
return '%dth' % n
if html:
return str(n) + '<sup>%s</sup>'% t[n % 10]
else:
return str(n) + t[n % 10]
def LineIndent(text, indent, maxwidth=None):
""" indent each new line with 'indent' """
if maxwidth:
parts = []
for part in text.split('\n'):
words = part.split(' ')
lines = []
tmpline = ''
for word in words:
if len(tmpline+' '+word) > maxwidth:
lines.append(tmpline.strip())
tmpline = word
else:
tmpline += ' ' + word
lines.append(tmpline.strip())
start = "\n%s"%indent
parts.append(indent + start.join(lines))
return "\n".join(parts)
else:
text = indent+text
text = text.replace('\n','\n%s'%indent)
return text
def safeId(id, nospaces=0, extra_allows=[]):
""" Just make sure it contains no dodgy characters """
lowercase = 'abcdefghijklmnopqrstuvwxyz'
digits = '0123456789'
specials = '_-.' + ''.join(extra_allows)
allowed = lowercase + lowercase.upper() + digits + specials
if not nospaces:
allowed = ' ' + allowed
n_id=[]
allowed_list = list(allowed)
for letter in list(id):
if letter in allowed_list:
n_id.append(letter)
return ''.join(n_id)
def banner(text, ch='=', length=78):
"""Return a banner line centering the given text.
"text" is the text to show in the banner. None can be given to have
no text.
"ch" (optional, default '=') is the banner line character (can
also be a short string to repeat).
"length" (optional, default 78) is the length of banner to make.
Examples:
>>> banner("Peggy Sue")
'================================= Peggy Sue =================================='
>>> banner("Peggy Sue", ch='-', length=50)
'------------------- Peggy Sue --------------------'
>>> banner("Pretty pretty pretty pretty Peggy Sue", length=40)
'Pretty pretty pretty pretty Peggy Sue'
"""
if text is None:
return ch * length
elif len(text) + 2 + len(ch)*2 > length:
# Not enough space for even one line char (plus space) around text.
return text
else:
remain = length - (len(text) + 2)
prefix_len = remain / 2
suffix_len = remain - prefix_len
if len(ch) == 1:
prefix = ch * prefix_len
suffix = ch * suffix_len
else:
prefix = ch * (prefix_len/len(ch)) + ch[:prefix_len%len(ch)]
suffix = ch * (suffix_len/len(ch)) + ch[:suffix_len%len(ch)]
return prefix + ' ' + text + ' ' + suffix
def normalizeMobileNumber(number, default_cc='44'):
""" prepare a mobile number """
number = number.strip()
if number.startswith('00%s'%default_cc):
number = "+"+number[2:]
elif number.startswith('0'):
number = '+%s'%default_cc+number[1:]
number = number.replace(' ','').replace('-','')
return number
def ss(s):
""" simple string """
return s.strip().lower()
def encodeEmailString(email, title=None, nolink=0):
""" just write the email like
<span class="aeh">peter_AT_peterbe_._com</span>
and a 'onLoad' Javascript will convert it to look nice.
The way we show the email address must match the Javascript
that converts it on the fly. """
methods = ['_dot_',' dot ', '_._']
shuffle(methods)
# replace . after @
if '@' in list(email):
afterbit = email.split('@')[1]
newbit = afterbit.replace('.', methods[0])
email = email.replace(afterbit, newbit)
methods = ['_', '~']
shuffle(methods)
atsigns = ['at','AT']
shuffle(atsigns)
# replace @ with *AT*
email = email.replace('@','%s%s%s'%(methods[0], atsigns[0], methods[0]))
if title is None or title == email:
title = email
spantag = '<span class="aeh">%s</span>'
spantag_link = '<span class="aeh"><a href="mailto:%s">%s</a></span>'
if nolink:
return spantag%email
else:
return spantag_link%(email, title)
def encodeEmailString2(email, title=None, nolink=0):
""" A second version of the old encodeEmailString() function.
This one returns a XHTML valid tag not wrapped in a <span> tag"""
methods = [urllib.quote(x) for x in ['_dot_',' dot ', '_._']]
shuffle(methods)
# replace . after @
if '@' in list(email):
afterbit = email.split('@')[1]
newbit = afterbit.replace('.', methods[0])
email = email.replace(afterbit, newbit)
methods = ['_', '~']
shuffle(methods)
atsigns = ['at','AT','ATT']
shuffle(atsigns)
r = random.randint(1,3)
if r==1:
email = email.replace('@','@(removethis)')
elif r==2:
email = email.replace('@','(removethis)@')
# replace @ with *AT*
email = email.replace('@','%s%s%s'%(methods[0], atsigns[0], methods[0]))
if title is None or title == email:
title = email
spantag = '<span class="aeh">%s</span>'
spantag_link = '<a class="aeh" href="mailto:%s">%s</a>'
if nolink:
return spantag % email
else:
return spantag_link % (email, title)
#-----------------------------------------------------------------------------
#
# Booleanism
#
def niceboolean(value):
if type(value) is bool:
return value
falseness = ('','no','off','false','none','0', 'f')
return str(value).lower().strip() not in falseness
def isOrdinalth(s):
""" return true if like 1st, 4th, 11th, 23rd """
ordn = re.compile(r"""\dst$
|\dnd$
|\drd$
|\dth$
|\d\dst$
|\d\dnd$
|\d\drd$
|\d\dth$""",
re.I|re.VERBOSE)
return not not ordn.findall(s)
def typelessEqual(a, b):
""" Checks equality of two values regardless of type """
if a is None or b is None:
# like Postgres deals with NULL
return False
try: # floatable?
a = float(a)
b = float(b)
return a == b
except ValueError:
try:
a = int(a)
b = int(b)
return a == b
except ValueError:
a = str(a)
b = str(b)
return a == b
def same_type(one, two):
""" use this because 'type' as variable can be used elsewhere """
return type(one)==type(two)
def safebool(value):
try:
return not not int(value)
except ValueError:
return 0
#-----------------------------------------------------------------------------
#
# Showers, converters, nicifiers
#
# taken from IssueTrackerProduct and modified
def ShowText(text, display_format='',
emaillinkfunction=None,
urllinkfunction=None,
allowhtml=False, # only applies to plaintext and structured_text
paragraphtag=True, # only applies with plaintext
):
"""
Display text, using harmless HTML
"""
if not text: # blank or None
return ""
if display_format == 'structuredtext':
#st=_replace_special_chars(text)
st=text
if not allowhtml:
for k,v in {'<':'&lt;', '>':'&gt;'}.items():
st = st.replace(k, v)
st = st.replace('[','|[|')
st = html_entity_fixer(st, skipchars=('"',))
st = structured_text(st)
if allowhtml:
for k,v in {'<':'&lt;', '>':'&gt;'}.items():
st = st.replace(v, k)
for k,v in {'&amp;lt;':'&lt;', '&amp;gt;':'&gt;',
'|[|':'['}.items():
st = st.replace(k,v)
st = addhrefs(st, emaillinkfunction=emaillinkfunction, urllinkfunction=urllinkfunction)
st = st.rstrip()
return st
elif display_format == 'html':
return text
else:
if paragraphtag:
t = '<p>%s</p>'%safe_html_quote(text)
else:
t = safe_html_quote(text)
t = t.replace('&amp;lt;','&lt;').replace('&amp;gt;','&gt;')
t = addhrefs(t)
t = newline_to_br(t)
return t
def ShowFilesize(bytes, whole_numbers=False):
""" Return nice representation of size """
if bytes < 1024:
return "1 Kb"
elif bytes > 1024**3:
if whole_numbers:
mb_bytes = '%0.02f'%(bytes / 1024.0**3)
else:
mb_bytes = '%.f'%(bytes / 1024.0**3)
return "%s Gb"%mb_bytes
elif bytes > 1024**2:
if whole_numbers:
mb_bytes = '%0.02f'%(bytes / 1024.0**2)
else:
mb_bytes = '%.f'%(bytes / 1024.0**2)
return "%s Mb"%mb_bytes
else:
return "%s Kb"%int(bytes / 1024)
def formatFloat(f, decimal_places=2):
""" return a float with thousand commas
and two decimal places if any. """
try:
assert type(f) is FloatType or type(f) is IntType, \
"parameter f is not float or int"
except AssertionError:
f = float(f)
if str(int(f))!=str(f): # try if f==1.1 but not if f==1
f= round(f, decimal_places)
_fmt = "%."+str(decimal_places)+"f"
f = _fmt%f
f = thousands_commas(f)
return f
def _QasList(q):
""" q is a string that might contain 'and' and/or 'or'.
Remove that and make it a list. """
L = []
for e in re.compile("'.*'|\".*\"").findall(q):
L.append(e[1:-1])
q = q.replace(e,'')
r=re.compile(r"\band\b|\bor\b", re.I)
return r.sub("", q).split() + L
def _warpLine(found, line, template):
# print found
return line
def highlightQ(q, text, template='<span class="highlight">%s</span>',
splitlines=False):
""" return a result text where certain words are highligheted.
This code is taken from the IssueTrackerProduct and modified """
if isinstance(q, str):
q = unicode(q, 'latin1')
if isinstance(text, str):
text = unicode(text, 'latin1')
if isinstance(template, str):
template = unicode(template, 'latin1')
transtab = string.maketrans('/ ','_ ')
#q=string.translate(q, transtab, '?&!;()<=>*#[]{}')
q = q.translate('?&!;()<=>*#[]{}')
highlighted_lines = []
template = " " + template.strip() + " "
text = " %s "%text
for e in _QasList(q):
regex = "(\s(%s)\s)" % e
merged_regex = re.compile(regex, re.I)
text = merged_regex.sub(template%r"\2", text)
if splitlines:
lines = []
for line in text.splitlines():
line_regex = re.compile('%s'%(template%'.*?'), re.MULTILINE|re.DOTALL)
if line_regex.findall(line):
lines.append(line)
return lines
return text.strip()
#-----------------------------------------------------------------------------
#
# Validators, checkers
#
def _ShouldBeNone(result): return result is not None
def _ShouldNotBeNone(result): return result is None
tests = (
# Thank you Bruce Eckels for these (some modifications by Peterbe)
(re.compile("^[0-9a-zA-Z\.\'\+\-\_]+\@[0-9a-zA-Z\.\-\_]+$"),
_ShouldNotBeNone, "Failed a"),
(re.compile("^[^0-9a-zA-Z]|[^0-9a-zA-Z]$"),
_ShouldBeNone, "Failed b"),
(re.compile("([0-9a-zA-Z\_]{1})\@."),
_ShouldNotBeNone, "Failed c"),
(re.compile(".\@([0-9a-zA-Z]{1})"),
_ShouldNotBeNone, "Failed d"),
(re.compile(".\.\-.|.\-\..|.\.\..|.\-\-."),
_ShouldBeNone, "Failed e"),
(re.compile(".\.\_.|.\-\_.|.\_\..|.\_\-.|.\_\_."),
_ShouldBeNone, "Failed f"),
(re.compile(".\.([a-zA-Z]{2,3})$|.\.([a-zA-Z]{2,4})$"),
_ShouldNotBeNone, "Failed g"),
# no underscore just left of @ sign or _ after the @
(re.compile("\_@|@[a-zA-Z0-9\-\.]*\_"),
_ShouldBeNone, "Failed h"),
)
def ValidEmailAddress(address, debug=None):
for test in tests:
if test[1](test[0].search(address)):
if debug: return test[2]
return 0
return 1
def ValidMobileNumber(mn):
""" Check if a mobile number is valid """
if mn is None:
return 0
mn = mn.replace(' ','').strip()
if len(mn) < 10:
return false
# | pipe means to "or" cases
# 1st: must start with '+' and be numbers the rest
# 2nd: must start with '0' and be numbers the rest
test = re.compile(r'^\+[0-9]+$|^0[0-9]+$')
return not not test.findall(mn) # use 'not not' to reduce to boolean
def ValidDate(day, month, year):
""" Checks if a date address is valid and within oferred range """
month = string.zfill(month, 2)
day = string.zfill(day, 2)
isodatestring = "%s-%s-%s"%(year, month, day)
try:
dateobj = DateTime(isodatestring)
return 1
except:
return 0
#-----------------------------------------------------------------------------
#
# Date testing, comparing, modifying
#
def somedate2DateTime(datestr, raiseerrors=1):
""" accept
today,
18/02/02,
18/02/02 15:00,
18/02/2003,
18/02/2003 15:00,
18,
18[st|nd|rd|th],
15:00,
18 15:00,
18 FeB[ruary],
18[st|nd|rd|th] FeB[ruary],
18 FeB[ruary] 2003,
18[st|nd|rd|th] FeB[ruary] 2003,
18 FeB[ruary] 15:00,
18[st|nd|rd|th] FeB[ruary] 15:00,
Fri[day],
Fri[day] 15:00
noW """
datestr = datestr.lower().strip()
datestr = _simpleSpace(datestr)
now = DateTime()
_months = ['January','February','March','April','May','June','July',
'August','September','October','November','December']
_months = [x.lower() for x in _months]
_d_months={}
for i in range(len(_months)):
_d_months[_months[i]] = i+1
_d_months[_months[i][:3]] = i+1
_months=_d_months
_days = ['Monday','Tuesday','Wednesday','Thursday',
'Friday','Saturday','Sunday']
_days = [x.lower() for x in _days]
_d_days={}
for e in _days:
_d_days[e]=e
_d_days[e[:3]]=e
_days=_d_days
def _onlyInt(*s, **kw):
for _s in s:
try:
_s=int(_s)
except ValueError:
if kw.get('pos_ord') and isOrdinalth(_s):
pass
else:
return 0
return 1
def _ord2Int(s):
""" convert 1st to 1 """
s = s.lower()
for each in ['st','nd','rd','th']:
s = s.replace(each,'')
return int(s)
def _possibleTime(s):
try:
h,m = s.split(':')
h=int(h)
h=int(m)
return 1
except:
return 0
def _possibleMonth(s, ms=_months):
return s.lower().strip() in ms.keys()
def _possibleDay(s, ds=_days):
return s.lower().strip() in ds.keys()
def _getmonthnr(s, ms=_months):
return ms[s.lower()]
def _getdayrepr(s, ds=_days):
return ds[s.lower().strip()]
# 'now'
if type(datestr) != type('s'):
# already a DateTime instance
return datestr
elif datestr == 'today':
return DateTime()
elif datestr == 'now':
return DateTime()
# '18/02/2003 13:30'
elif datestr.count('/') == 2 and datestr.find(' ') > -1 and \
datestr.split(' ')[1].find(':') > -1:
yy, mm, dd = twistDateformat(datestr.split(' ')[0]).split('/')
time = datestr.split(' ')[1]
if time.count(':') == 1:
hour, minute = time.split(':')
hour = int(hour)
minute = int(minute)
datestring = '%s/%s/%s %s:%s'%(yy, mm, dd, hour, minute)
return DateTime(datestring)
# '18/02/2003'
elif datestr.count('/') == 2:
yy, mm, dd = twistDateformat(datestr).split('/')
dd=int(dd)
mm=int(mm)
yy=int(yy)
datestring = '%s/%s/%s 00:00'%(yy,mm,dd)
return DateTime(datestring)
# '18' or '18[st|nd|rd|th]'
elif _onlyInt(datestr, pos_ord=1):
dd = int(_ord2Int(datestr))
mm = int(now.strftime('%m'))
yy = int(now.strftime('%Y'))
datestring = '%s/%s/%s 00:00'%(yy, mm, dd)
return DateTime(datestring)
# '15:00'
elif len(datestr.split(' '))==1 and _possibleTime(datestr):
dd = int(now.strftime('%d'))
mm = int(now.strftime('%m'))
yy = int(now.strftime('%Y'))
h, m = datestr.split(':')
datestring = '%s/%s/%s %s:%s'%(yy,mm,dd, h, m)
return DateTime(datestring)
# '18 15:00' or '18[st|nd|rd|th] 15:00'
elif len(datestr.split(' '))==2 and\
_onlyInt(datestr.split(' ')[0], pos_ord=1) and \
_possibleTime(datestr.split(' ')[1]):
dd = int(_ord2Int(datestr.split(' ')[0]))
mm = int(now.strftime('%m'))
yy = int(now.strftime('%Y'))
h, m = datestr.split(' ')[1].split(':')
datestring = '%s/%s/%s %s:%s'%(yy,mm,dd,h,m)
return DateTime(datestring)
# '15 FeB[ruary] 2003' or '15[st|nd|rd|th] FeB[ruary] 2003'
elif len(datestr.split(' '))==3 and\
_onlyInt(datestr.split(' ')[0], pos_ord=1) and\
_possibleMonth(datestr.split(' ')[1]) and \
_onlyInt(datestr.split(' ')[2]):
splitted = datestr.split(' ')
dd = int(_ord2Int(splitted[0]))
mm = _getmonthnr(splitted[1])
yy = int(splitted[2])
datestring = '%s/%s/%s 00:00'%(yy,mm,dd)
return DateTime(datestring)
# '15 Feb' or '15[st|nd|rd|th] Feb'
elif len(datestr.split(' '))==2 and \
_onlyInt(datestr.split(' ')[0], pos_ord=1) and\
_possibleMonth(datestr.split(' ')[1]):
dd = int(_ord2Int(datestr.split(' ')[0]))
mm = _getmonthnr(datestr.split(' ')[1])
yy = int(now.strftime('%Y'))
datestring = '%s/%s/%s 00:00'%(yy,mm,dd)
return DateTime(datestring)
# '18[st|nd|rd|th] FeB[ruary] 15:00'
elif len(datestr.split(' '))==3 and \
_onlyInt(datestr.split(' ')[0], pos_ord=1) and \
_possibleMonth(datestr.split(' ')[1]) and \
_possibleTime(datestr.split(' ')[2]):
dd = int(_ord2Int(datestr.split(' ')[0]))
mm = _getmonthnr(datestr.split(' ')[1])
yy = int(now.strftime('%Y'))
h, m = datestr.split(' ')[2].split(':')
datestring = '%s/%s/%s %s:%s'%(yy,mm,dd,h,m)
return DateTime(datestring)
# 'Fri[day]'
elif _possibleDay(datestr):
day = _getdayrepr(datestr)
fromdate = DateTime(now.strftime('%Y/%m/%d 00:00'))+1
return _getNextDayByDay(day, fromdate)
# 'Fri[day] 15:00'
elif len(datestr.split(' '))==2 and _possibleDay(datestr.split(' ')[0])\
and _possibleTime(datestr.split(' ')[1]):
day = _getdayrepr(datestr.split(' ')[0])
h, m = datestr.split(' ')[1].split(':')
_fmt = '%Y/%m/%d '+'%s:%s'%(h,m)
fromdate = DateTime(now.strftime(_fmt))+1
return _getNextDayByDay(day, fromdate)
else:
if raiseerrors:
raise "SomeDateFailed", datestr
else:
return None
def _getNextDayByDay(day, fromdate):
""" loop through all dates to find next match """
for i in range(8):
then = fromdate+i
if then.strftime('%A').lower() == day:
return then
return None
# Language constants
MINUTE = 'minute'
MINUTES = 'minutes'
HOUR = 'hour'
HOURS = 'hours'
YEAR = 'year'
YEARS = 'years'
MONTH = 'month'
MONTHS = 'months'
WEEK = 'week'
WEEKS = 'weeks'
DAY = 'day'
DAYS = 'days'
AND = 'and'
def timeSince(firstdate, seconddate, afterword=None,
minute_granularity=False,
max_no_sections=3):
"""
Use two date objects to return in plain english the difference between them.
E.g. "3 years and 2 days"
or "1 year and 3 months and 1 day"
Try to use weeks when the no. of days > 7
If less than 1 day, return number of hours.
If there is "no difference" between them, return false.
"""
def wrap_afterword(result, afterword=afterword):
if afterword is not None:
return "%s %s" % (result, afterword)
else:
return result
fdo = firstdate
sdo = seconddate
day_difference = int(abs(sdo-fdo))
years = day_difference/365
months = (day_difference % 365)/30
days = (day_difference % 365) % 30
minutes = ((day_difference % 365) % 30) % 24
if days == 0 and months == 0 and years == 0:
# use hours
hours=int(round(abs(sdo-fdo)*24, 2))
if hours == 1:
return wrap_afterword("1 %s" % (HOUR))
elif hours > 0:
return wrap_afterword("%s %s" % (hours, HOURS))
elif minute_granularity:
minutes = int(round(abs(sdo-fdo) * 24 * 60, 3))
if minutes == 1:
return wrap_afterword("1 %s" % MINUTE)
elif minutes > 0:
return wrap_afterword("%s %s" % (minutes, MINUTES))
else:
# if the differnce is smaller than 1 minute,
# return 0.
return 0
else:
# if the difference is smaller than 1 hour,
# return it false
return 0
else:
s = []
if years == 1:
s.append('1 %s'%(YEAR))
elif years > 1:
s.append('%s %s'%(years,YEARS))
if months == 1:
s.append('1 %s'%MONTH)
elif months > 1:
s.append('%s %s'%(months,MONTHS))
if days == 1:
s.append('1 %s'%DAY)
elif days == 7:
s.append('1 %s'%WEEK)
elif days == 14:
s.append('2 %s'%WEEKS)
elif days == 21:
s.append('3 %s'%WEEKS)
elif days > 14:
weeks = days / 7
days = days % 7
if weeks == 1:
s.append('1 %s'%WEEK)
else:
s.append('%s %s'%(weeks, WEEKS))
if days % 7 == 1:
s.append('1 %s'%DAY)
elif days > 0:
s.append('%s %s'%(days % 7,DAYS))
elif days > 1:
s.append('%s %s'%(days,DAYS))
s = s[:max_no_sections]
if len(s)>1:
return wrap_afterword("%s" % (string.join(s,' %s '%AND)))
else:
return wrap_afterword("%s" % s[0])
MONTHS_ABBREVIATED = [date(2007, e, 1).strftime('%b').lower() for e in range(1,13)]
MONTHS_FULL = [date(2007, e, 1).strftime('%B').lower() for e in range(1,13)]
LAZY_MONTH_YEAR_REGEX = re.compile(
'^(%s)\s+(%s)$' % ('|'.join(MONTHS_ABBREVIATED+MONTHS_FULL),
'|'.join([str(x) for x in range(2004, 2015)])),
re.I)
LAZY_YEAR_MONTH_REGEX = re.compile(
'^(%s)\s+(%s)$' % ('|'.join([str(x) for x in range(2004, 2015)]),
'|'.join(MONTHS_ABBREVIATED+MONTHS_FULL)),
re.I)
LAZY_DAY_MONTH_YEARLESS_REGEX = re.compile(
'^(\d{1,2})\s+(%s)$' % ('|'.join(MONTHS_FULL+MONTHS_ABBREVIATED)),
re.I)
LAZY_MONTH_DAY_YEARLESS_REGEX = re.compile(
'^(%s)\s+(\d{1,2})$' % ('|'.join(MONTHS_FULL+MONTHS_ABBREVIATED)),
re.I)
def lazyDateThisYear(datestr):
""" return and change the datestr we recognize it to be one of these
kinds of formats:
'20 June'
'20 Jun'
'June 20'
'Jun 20'
If we do recognize it, return it with todays year appended at the
end.
"""
if LAZY_DAY_MONTH_YEARLESS_REGEX.findall(datestr):
day, month = LAZY_DAY_MONTH_YEARLESS_REGEX.findall(datestr)[0]
today = DateTime()
return '%s %s %s' %(day, month, today.strftime('%Y'))
elif LAZY_MONTH_DAY_YEARLESS_REGEX.findall(datestr):
month, day = LAZY_MONTH_DAY_YEARLESS_REGEX.findall(datestr)[0]
today = DateTime()
return '%s %s %s' %(day, month, today.strftime('%Y'))
return datestr
def lazyDateDayless(datestr, last_day_of_month=False):
""" if datestr is 'june 2005' return '2005/06/01'
(or '2005/06/30' if last_day_of_month is True)
"""
def fixDateStr(datestr, month, year, last_day_of_month):
try:
month = [x.lower() for x in MONTHS_FULL].index(month.lower()) + 1
except ValueError:
month = [x.lower() for x in MONTHS_ABBREVIATED].index(month.lower()) + 1
if last_day_of_month:
year = int(year)
if int(month) + 1 > 12:
year += (int(month) + 1) % 12
month = (int(month) + 1) / 12
else:
month = int(month) + 1
datestr = '%s/%s/01' % (year, string.zfill(month, 2))
datestr = (DateTime(datestr)-1).strftime('%Y/%m/%d')
else:
datestr = '%s/%s/01' % (year, string.zfill(month, 2))
return datestr
if LAZY_MONTH_YEAR_REGEX.findall(datestr):
month, year = LAZY_MONTH_YEAR_REGEX.findall(datestr)[0]
datestr = fixDateStr(datestr, month, year, last_day_of_month)
elif LAZY_YEAR_MONTH_REGEX.findall(datestr):
year, month = LAZY_YEAR_MONTH_REGEX.findall(datestr)[0]
datestr = fixDateStr(datestr, month, year, last_day_of_month)
elif datestr.lower() in [x.lower() for x in MONTHS_FULL]:
datestr += ' ' + strftime('%Y')
return lazyDateDayless(datestr)
elif datestr.lower() in [x.lower() for x in MONTHS_ABBREVIATED]:
full_index = [x.lower() for x in MONTHS_ABBREVIATED].index(datestr.lower())
full = MONTHS_FULL[full_index]
datestr = '%s ' % full + strftime('%Y')
return lazyDateDayless(datestr)
return datestr
#-----------------------------------------------------------------------------
#
# External programs
#
try:
import subprocess
common_version_arguments = {
'less':'-V',
'convert':'-version',
'svn':'--version',
'dot':'-V',
'ocrad':'-h',
'pngtopnm':'-h',
'djpeg':'-h',
}
def hasExternalProgram(name, cmd_argument_version=None):
if cmd_argument_version is None:
cmd_argument_version = common_version_arguments.get(name, '-v')
proc = subprocess.Popen('%s %s' % (name, cmd_argument_version),
shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
v = proc.communicate()
# some programs sends the version info on stderr
if name in ('dot','pdftotext','pngtopnm','djpeg'):
return bool(v[1])
else:
return not bool(v[1])
except ImportError:
def hasExternalProgram(name, version=None):
print >>sys.stderr, "hasExternalProgram() needs subprocess which is "\
"only available in python 2.4 and newer"
return False
#-----------------------------------------------------------------------------
#
# System related
#
def getEnvBool(key, default):
""" return an boolean from the environment variables """
value = os.environ.get(key, default)
try:
value = not not int(value)
except ValueError:
if str(value).lower().strip() in ['yes','on','t','y']:
value = 1
elif str(value).lower().strip() in ['no','off','f','n']:
value = 0
else:
value = default
return value
def getEnvInt(key, default):
""" return an integer from the environment variables """
value = os.environ.get(key, default)
try:
return int(value)
except ValueError:
return default
def getEnvStr(key, default=''):
""" return an integer from the environment variables """
value = os.environ.get(key, default)
try:
return str(value)
except ValueError:
return default
def test_highlightQ():
text = '''this is Peter Bengtsson
is here
this line has nothing that we care about
bu here peter is mentioned
again'''
print highlightQ("peter is", text, splitlines=1)
print
text = "Expertise"
print highlightQ("expertise or nothing", text, template="<s>%s</s>")
def test_ShowText():
text = "Peter **bengtsson** was _here_"
# print ShowText(text, 'structuredtext')
text += " <keep>"
#print ShowText(text, 'structuredtext')
#print ShowText(text, 'structuredtext', allowhtml=1)
text = "Word\n\nWord2"
print ShowText(text)
if __name__=='__main__':
#test_highlightQ()
test_ShowText()
Jump to Line
Something went wrong with that request. Please try again.