Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ language: python
python:
- 3.6
- 3.7
- 3.8

branches:
only:
Expand Down
19 changes: 14 additions & 5 deletions legi/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import re
from sqlite3 import Connection, IntegrityError, OperationalError, ProgrammingError, Row
import sre_parse
import sys
import traceback
from unicodedata import combining, decomposition, normalize

Expand Down Expand Up @@ -199,6 +200,14 @@ def group_by_2(iterable):
yield (a, b)


class _Tokenizer(sre_parse.Tokenizer):

if sys.version_info < (3, 8, 0):
# Prior to Python 3.8 the `getuntil` method didn't have the `name` argument
def getuntil(self, terminator, name):
return super(_Tokenizer, self).getuntil(terminator)


def add_accentless_fallbacks(pattern):
r"""Modifies a regexp pattern to also match accentless text.

Expand All @@ -223,7 +232,7 @@ def remove_accent(c):
return chr(int(decomposition(c).split(' ', 1)[0], 16))

r = []
source = sre_parse.Tokenizer(pattern)
source = _Tokenizer(pattern)
sourceget = source.get
while True:
this = source.next
Expand Down Expand Up @@ -263,16 +272,16 @@ def remove_accent(c):
elif this == 'P':
if source.next == '<':
# named group
this += source.getuntil('>') + '>'
this += source.getuntil('>', 'group name') + '>'
elif source.next == '=':
# named backreference
this += source.getuntil(')') + ')'
this += source.getuntil(')', 'group name') + ')'
elif this == '#':
# comment
this += source.getuntil(')') + ')'
this += source.getuntil(')', 'comment') + ')'
elif this == '(':
# conditional backreference group
this += source.getuntil(')') + ')'
this += source.getuntil(')', 'group name') + ')'
r.append('(?' + this)
else:
if decomposition(this):
Expand Down