-
-
Notifications
You must be signed in to change notification settings - Fork 29
/
unicode_to_ascii.py
executable file
·72 lines (56 loc) · 1.98 KB
/
unicode_to_ascii.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Unicode to ASCII Utility
========================
"""
import sys
if sys.version_info[0] < 3:
# Smelly hack for Python 2.x: https://stackoverflow.com/q/3828723/931625
reload(sys) # noqa
sys.setdefaultencoding('utf-8')
import codecs # noqa
import os # noqa
import unicodedata # noqa
__copyright__ = 'Copyright (C) 2018-2020 - Colour Developers'
__license__ = 'New BSD License - https://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-developers@colour-science.org'
__status__ = 'Production'
__all__ = ['SUBSTITUTIONS', 'unicode_to_ascii']
SUBSTITUTIONS = {
'–': '-',
'“': '"',
'”': '"',
'‘': "'",
'’': "'",
'′': "'",
}
def unicode_to_ascii(root_directory):
"""
Recursively converts from unicode to ASCII *.py*, *.bib* and *.rst* files
in given directory.
Parameters
----------
root_directory : unicode
Directory to convert the files from unicode to ASCII.
"""
for root, dirnames, filenames in os.walk(root_directory):
for filename in filenames:
if (not filename.endswith('.tex') and
not filename.endswith('.py') and
not filename.endswith('.bib') and
not filename.endswith('.rst')):
continue
if filename == 'unicode_to_ascii.py':
continue
filename = os.path.join(root, filename)
with codecs.open(filename, encoding='utf8') as file_handle:
content = file_handle.read()
with codecs.open(filename, 'w', encoding='utf8') as file_handle:
for key, value in SUBSTITUTIONS.items():
content = content.replace(key, value)
content = unicodedata.normalize('NFD', content)
file_handle.write(content)
if __name__ == '__main__':
unicode_to_ascii(os.path.join('..', 'colour_checker_detection'))