Add brownie.text.transliterate

CHANGELOG.rst

@@ -15,6 +15,7 @@ features.
 - Added `doc` parameter to :func:`brownie.datastructures.sequences.namedtuple`.
 - Added :mod:`brownie.context`.
 - Added :func:`brownie.functional.fmap`.
+- Added :mod:`brownie.text`.
 
 0.5.1
 -----

brownie/tests/__init__.py

@@ -10,6 +10,7 @@
 
 
 all = Tests('brownie.tests.%s.tests' % module for module in [
+    'text',
     'itools',
     'caching',
     'proxies',

brownie/tests/text.py

@@ -0,0 +1,36 @@
+# coding: utf-8
+"""
+    brownie.tests.text
+    ~~~~~~~~~~~~~~~~~~
+
+    Tests for :mod:`brownie.tests`.
+
+    :copyright: 2010-2011 by Daniel Neuhäuser
+    :license: BSD, see LICENSE.rst for details
+"""
+try:
+    import translitcodec
+except ImportError:
+    translitcodec = None
+from attest import Tests, Assert
+
+from brownie.text import transliterate
+
+
+tests = Tests()
+
+
+@tests.test
+def test_transliterate():
+    Assert(transliterate(u'äöü', 'one')) == u'aou'
+
+    tests = zip(
+        [
+            (u'©', 'long'),
+            (u'©', 'short'),
+            (u'☺', 'one'),
+        ],
+        [u''] * 3 if translitcodec is None else [u'(c)', u'c', u'?']
+    )
+    for args, result in tests:
+        Assert(transliterate(*args)) == result

brownie/text.py

@@ -0,0 +1,51 @@
+# coding: utf-8
+"""
+    brownie.text
+    ~~~~~~~~~~~~
+
+    Utilities to deal with text.
+
+    .. versionadded:: 0.6
+
+    :copyright: 2010-2011 by Daniel Neuhäuser
+    :license: BSD, see LICENSE.rst for details
+"""
+import unicodedata
+
+try:
+    import translitcodec
+except ImportError: # pragma: no cover
+    translitcodec = None
+
+
+def transliterate(string, length='long'):
+    """
+    Returns a transliterated version of the given unicode `string`.
+
+    By specifying `length` you can specify how many characters are used for a
+    replacement:
+
+    `long`
+        Use as many characters as needed to make a natural replacement.
+
+    `short`
+        Use as few characters as possible to make a replacement.
+
+    `one`
+        Use only one character to make a replacement. If a character cannot
+        be transliterated with a single character replace it with `'?'`.
+
+    If available translitcodec_ is used, which provides more natural results.
+
+    .. _translitcodec: http://pypi.python.org/pypi/translitcodec
+    """
+    if length not in ('long', 'short', 'one'):
+        raise ValueError('unknown length: %r' % length)
+    if translitcodec is None:
+        return unicodedata.normalize('NFKD', string) \
+            .encode('ascii', 'ignore') \
+            .decode('ascii')
+    else:
+        if length == 'one':
+            return string.encode('translit/one/ascii', 'replace').decode('ascii')
+        return string.encode('translit/%s' % length)

docs/api/text.rst

@@ -0,0 +1,9 @@
+.. module:: brownie.text
+
+Text
+====
+
+This module provides utilities to deal with text.
+
+
+.. autofunction:: transliterate

docs/index.rst

@@ -24,6 +24,7 @@ API Documentation
    api/proxies
    api/terminal
    api/context
+   api/text
 
 Developer Documentation
 -----------------------

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = docs,py25,py26,py27,pypy,eventlet,py26-eventlet,py27-eventlet
+envlist = docs,py25,py26,py27,pypy,eventlet,py26-eventlet,py27-eventlet,translitcodec,py26-translitcodec,py27-translitcodec
 
 [testenv]
 deps =
@@ -11,9 +11,16 @@ commands =
 deps =
  attest==0.4
  eventlet
-commands=
+commands =
  python runtests.py context
 
+[testenv:translitcodec]
+deps =
+ attest==0.4
+ translitcodec
+commands =
+ python runtests.py text
+
 [testenv:py26]
 deps =
  attest==0.4
@@ -25,12 +32,19 @@ commands =
  sphinx-build -W -b doctest docs docs/_build/doctest
 
 [testenv:py26-eventlet]
-deps=
+deps =
  attest==0.4
  eventlet
 commands =
  python runtests.py context
 
+[testenv:py26-translitcodec]
+deps =
+ attest==0.4
+ translitcodec
+commands =
+ python runtests.py text
+
 [testenv:py27]
 deps =
  attest==0.4
@@ -48,6 +62,13 @@ deps =
 commands =
  python runtests.py context
 
+[testenv:py27-translitcodec]
+deps =
+ attest==0.4
+ translitcodec
+commands =
+ python runtests.py text
+
 [testenv:docs]
 deps =
  sphinx