Skip to content
This repository was archived by the owner on Jul 7, 2023. It is now read-only.

Simplify native_to_unicode() & unicode_to_native() #243

Merged
merged 2 commits into from
Aug 25, 2017
Merged

Simplify native_to_unicode() & unicode_to_native() #243

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d48d35
Simplify native_to_unicode() & unicode_to_native()
Aug 23, 2017
33e798a
A much cleaner approach
Aug 23, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions tensor2tensor/data_generators/text_encoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,19 +56,13 @@
_ESCAPE_CHARS = set(u"\\_u;0123456789")


def native_to_unicode_py2(s):
"""Python 2: transform native string to Unicode."""
return s if isinstance(s, unicode) else s.decode("utf8")


# Conversion between Unicode and UTF-8, if required (on Python2)
if six.PY2:
native_to_unicode = native_to_unicode_py2
unicode_to_native = lambda s: s.encode("utf-8")
def native_to_unicode(s): return s if isinstance(s, unicode) else s.decode("utf8") # noqa: F821
def unicode_to_native(s): return s.encode("utf-8")
else:
# No conversion required on Python3
native_to_unicode = lambda s: s
unicode_to_native = lambda s: s
# No conversion required on Python >= 3
def native_to_unicode(s): return s
def unicode_to_native(s): return s


class TextEncoder(object):
Expand Down