bpo-13153: Use OS native encoding for converting between Python and T…

…cl. (GH-16545)

On Windows use UTF-16 (or UTF-32 for 32-bit Tcl_UniChar) with the
"surrogatepass" error handler for converting to/from Tcl Unicode objects.

On Linux use UTF-8 with the "surrogateescape" error handler for converting
to/from Tcl String objects.

Converting strings from Tcl to Python and back now never fails
(except MemoryError).
(cherry picked from commit 06cb94b)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

Lib/idlelib/editor.py

@@ -358,21 +358,6 @@ def set_width(self):
             Font(text, font=text.cget('font')).measure('0')
         self.width = pixel_width // zero_char_width
 
-    def _filename_to_unicode(self, filename):
-        """Return filename as BMP unicode so displayable in Tk."""
-        # Decode bytes to unicode.
-        if isinstance(filename, bytes):
-            try:
-                filename = filename.decode(self.filesystemencoding)
-            except UnicodeDecodeError:
-                try:
-                    filename = filename.decode(self.encoding)
-                except UnicodeDecodeError:
-                    # byte-to-byte conversion
-                    filename = filename.decode('iso8859-1')
-        # Replace non-BMP char with diamond questionmark.
-        return re.sub('[\U00010000-\U0010FFFF]', '\ufffd', filename)
-
     def new_callback(self, event):
         dirname, basename = self.io.defaultfilename()
         self.flist.new(dirname)
@@ -963,10 +948,8 @@ def update_recent_files_list(self, new_file=None):
             menu.delete(0, END)  # clear, and rebuild:
             for i, file_name in enumerate(rf_list):
                 file_name = file_name.rstrip()  # zap \n
-                # make unicode string to display non-ASCII chars correctly
-                ufile_name = self._filename_to_unicode(file_name)
                 callback = instance.__recent_file_callback(file_name)
-                menu.add_command(label=ulchars[i] + " " + ufile_name,
+                menu.add_command(label=ulchars[i] + " " + file_name,
                                  command=callback,
                                  underline=0)
 
@@ -1004,16 +987,10 @@ def reset_undo(self):
 
     def short_title(self):
         filename = self.io.filename
-        if filename:
-            filename = os.path.basename(filename)
-        else:
-            filename = "untitled"
-        # return unicode string to display non-ASCII chars correctly
-        return self._filename_to_unicode(filename)
+        return os.path.basename(filename) if filename else "untitled"
 
     def long_title(self):
-        # return unicode string to display non-ASCII chars correctly
-        return self._filename_to_unicode(self.io.filename or "")
+        return self.io.filename or ""
 
     def center_insert_event(self, event):
         self.center()

Lib/idlelib/idle_test/test_editor.py

@@ -30,18 +30,6 @@ def test_init(self):
         e._close()
 
 
-class EditorFunctionTest(unittest.TestCase):
-
-    def test_filename_to_unicode(self):
-        func = Editor._filename_to_unicode
-        class dummy():
-            filesystemencoding = 'utf-8'
-        pairs = (('abc', 'abc'), ('a\U00011111c', 'a\ufffdc'),
-                 (b'abc', 'abc'), (b'a\xf0\x91\x84\x91c', 'a\ufffdc'))
-        for inp, out in pairs:
-            self.assertEqual(func(dummy, inp), out)
-
-
 class TestGetLineIndent(unittest.TestCase):
     def test_empty_lines(self):
         for tabwidth in [1, 2, 4, 6, 8]:

Lib/idlelib/pyshell.py

@@ -679,14 +679,6 @@ def runsource(self, source):
         self.more = 0
         # at the moment, InteractiveInterpreter expects str
         assert isinstance(source, str)
-        #if isinstance(source, str):
-        #    from idlelib import iomenu
-        #    try:
-        #        source = source.encode(iomenu.encoding)
-        #    except UnicodeError:
-        #        self.tkconsole.resetoutput()
-        #        self.write("Unsupported characters in input\n")
-        #        return
         # InteractiveInterpreter.runsource() calls its runcode() method,
         # which is overridden (see below)
         return InteractiveInterpreter.runsource(self, source, filename)
@@ -1298,16 +1290,6 @@ def resetoutput(self):
         self.set_line_and_column()
 
     def write(self, s, tags=()):
-        if isinstance(s, str) and len(s) and max(s) > '\uffff':
-            # Tk doesn't support outputting non-BMP characters
-            # Let's assume what printed string is not very long,
-            # find first non-BMP character and construct informative
-            # UnicodeEncodeError exception.
-            for start, char in enumerate(s):
-                if char > '\uffff':
-                    break
-            raise UnicodeEncodeError("UCS-2", char, start, start+1,
-                                     'Non-BMP character not supported in Tk')
         try:
             self.text.mark_gravity("iomark", "right")
             count = OutputWindow.write(self, s, tags, "iomark")

Lib/idlelib/runscript.py

@@ -147,8 +147,7 @@ def _run_module_event(self, event, *, customize=False):
         interp = self.shell.interp
         if pyshell.use_subprocess and restart:
             interp.restart_subprocess(
-                    with_cwd=False, filename=
-                    self.editwin._filename_to_unicode(filename))
+                    with_cwd=False, filename=filename)
         dirname = os.path.dirname(filename)
         argv = [filename]
         if self.cli_args:

Lib/test/test_tcl.py

@@ -429,9 +429,12 @@ def passValue(value):
         self.assertEqual(passValue(False), False if self.wantobjects else '0')
         self.assertEqual(passValue('string'), 'string')
         self.assertEqual(passValue('string\u20ac'), 'string\u20ac')
+        self.assertEqual(passValue('string\U0001f4bb'), 'string\U0001f4bb')
         self.assertEqual(passValue('str\x00ing'), 'str\x00ing')
         self.assertEqual(passValue('str\x00ing\xbd'), 'str\x00ing\xbd')
         self.assertEqual(passValue('str\x00ing\u20ac'), 'str\x00ing\u20ac')
+        self.assertEqual(passValue('str\x00ing\U0001f4bb'),
+                         'str\x00ing\U0001f4bb')
         self.assertEqual(passValue(b'str\x00ing'),
                          b'str\x00ing' if self.wantobjects else 'str\x00ing')
         self.assertEqual(passValue(b'str\xc0\x80ing'),
@@ -490,6 +493,7 @@ def float_eq(actual, expected):
         check('string')
         check('string\xbd')
         check('string\u20ac')
+        check('string\U0001f4bb')
         check('')
         check(b'string', 'string')
         check(b'string\xe2\x82\xac', 'string\xe2\x82\xac')
@@ -531,6 +535,7 @@ def test_splitlist(self):
             ('a\n b\t\r c\n ', ('a', 'b', 'c')),
             (b'a\n b\t\r c\n ', ('a', 'b', 'c')),
             ('a \u20ac', ('a', '\u20ac')),
+            ('a \U0001f4bb', ('a', '\U0001f4bb')),
             (b'a \xe2\x82\xac', ('a', '\u20ac')),
             (b'a\xc0\x80b c\xc0\x80d', ('a\x00b', 'c\x00d')),
             ('a {b c}', ('a', 'b c')),

Lib/tkinter/test/test_tkinter/test_misc.py

@@ -156,6 +156,28 @@ def callback():
         with self.assertRaises(tkinter.TclError):
             root.tk.call('after', 'info', idle1)
 
+    def test_clipboard(self):
+        root = self.root
+        root.clipboard_clear()
+        root.clipboard_append('Ùñî')
+        self.assertEqual(root.clipboard_get(), 'Ùñî')
+        root.clipboard_append('çōđě')
+        self.assertEqual(root.clipboard_get(), 'Ùñîçōđě')
+        root.clipboard_clear()
+        with self.assertRaises(tkinter.TclError):
+            root.clipboard_get()
+
+    def test_clipboard_astral(self):
+        root = self.root
+        root.clipboard_clear()
+        root.clipboard_append('𝔘𝔫𝔦')
+        self.assertEqual(root.clipboard_get(), '𝔘𝔫𝔦')
+        root.clipboard_append('𝔠𝔬𝔡𝔢')
+        self.assertEqual(root.clipboard_get(), '𝔘𝔫𝔦𝔠𝔬𝔡𝔢')
+        root.clipboard_clear()
+        with self.assertRaises(tkinter.TclError):
+            root.clipboard_get()
+
 
 tests_gui = (MiscTest, )
 

Lib/tkinter/test/test_ttk/test_widgets.py

@@ -489,8 +489,7 @@ def check_get_current(getval, currval):
                         expected=('mon', 'tue', 'wed', 'thur'))
         self.checkParam(self.combo, 'values', ('mon', 'tue', 'wed', 'thur'))
         self.checkParam(self.combo, 'values', (42, 3.14, '', 'any string'))
-        self.checkParam(self.combo, 'values', '',
-                        expected='' if get_tk_patchlevel() < (8, 5, 10) else ())
+        self.checkParam(self.combo, 'values', '')
 
         self.combo['values'] = ['a', 1, 'c']
 
@@ -1245,12 +1244,7 @@ def test_values(self):
                         expected=('mon', 'tue', 'wed', 'thur'))
         self.checkParam(self.spin, 'values', ('mon', 'tue', 'wed', 'thur'))
         self.checkParam(self.spin, 'values', (42, 3.14, '', 'any string'))
-        self.checkParam(
-            self.spin,
-            'values',
-            '',
-            expected='' if get_tk_patchlevel() < (8, 5, 10) else ()
-        )
+        self.checkParam(self.spin, 'values', '')
 
         self.spin['values'] = ['a', 1, 'c']
 
@@ -1308,8 +1302,7 @@ def test_columns(self):
         self.checkParam(widget, 'columns', 'a b c',
                         expected=('a', 'b', 'c'))
         self.checkParam(widget, 'columns', ('a', 'b', 'c'))
-        self.checkParam(widget, 'columns', (),
-                        expected='' if get_tk_patchlevel() < (8, 5, 10) else ())
+        self.checkParam(widget, 'columns', '')
 
     def test_displaycolumns(self):
         widget = self.create()

Misc/NEWS.d/next/Library/2019-09-29-22-47-37.bpo-13153.0mO9qR.rst

@@ -0,0 +1,4 @@
+OS native encoding is now used for converting between Python strings and
+Tcl objects.  This allows to display, copy and paste to clipboard emoji and
+other non-BMP characters.  Converting strings from Tcl to Python and back
+now never fails (except MemoryError).