-
Notifications
You must be signed in to change notification settings - Fork 119
/
test_characters.py
57 lines (43 loc) · 1.79 KB
/
test_characters.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
from ftfy import (
fix_encoding, fix_encoding_and_explain, fix_text, fix_and_explain, apply_plan
)
from ftfy.fixes import remove_control_chars, fix_surrogates
from ftfy.chardata import possible_encoding
from ftfy.badness import badness
import unicodedata
import sys
def test_possible_encoding():
for codept in range(256):
char = chr(codept)
assert possible_encoding(char, 'latin-1')
def test_byte_order_mark():
assert fix_encoding('') == '\ufeff'
def test_control_chars():
text = (
"\ufeffSometimes, \ufffcbad ideas \x7f\ufffalike these characters\ufffb "
"\u206aget standardized.\r\n"
)
fixed = "Sometimes, bad ideas like these characters get standardized.\r\n"
assert remove_control_chars(text) == fixed
def test_welsh_flag():
# ftfy used to remove "tag characters", but they have been repurposed in the
# "Flag of England", "Flag of Scotland", and "Flag of Wales" emoji sequences.
text = "This flag has a dragon on it 🏴"
assert remove_control_chars(text) == text
def test_ohio_flag():
# I did not expect to find the "Flag of Ohio" emoji in the wild but there it is.
# Test that this emoji (which no emoji database believes has been implemented)
# passes through unchanged.
text = "#superman #ohio 🏴\U000e0075\U000e0073\U000e006f\U000e0068\U000e007f #cleveland #usa 🇺🇸"
assert fix_text(text) == text
def test_surrogates():
assert fix_surrogates('\udbff\udfff') == '\U0010ffff'
assert fix_surrogates('\ud800\udc00') == '\U00010000'
def test_color_escapes():
fixed, plan = fix_and_explain("\001\033[36;44mfoo")
print(plan)
assert fixed == "foo"
assert plan == [
("apply", "remove_terminal_escapes"),
("apply", "remove_control_chars")
]