Update {test_}json from CPython 3.10.5

Author: CPython developers

Lib/json/__init__.py

@@ -133,7 +133,7 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
@@ -195,7 +195,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
@@ -329,8 +329,6 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None,
 
     To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
     kwarg; otherwise ``JSONDecoder`` is used.
-
-    The ``encoding`` argument is ignored and deprecated since Python 3.1.
     """
     if isinstance(s, str):
         if s.startswith('\ufeff'):
@@ -342,15 +340,6 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None,
                             f'not {s.__class__.__name__}')
         s = s.decode(detect_encoding(s), 'surrogatepass')
 
-    if "encoding" in kw:
-        import warnings
-        warnings.warn(
-            "'encoding' is ignored and deprecated. It will be removed in Python 3.9",
-            DeprecationWarning,
-            stacklevel=2
-        )
-        del kw['encoding']
-
     if (cls is None and object_hook is None and
             parse_int is None and parse_float is None and
             parse_constant is None and object_pairs_hook is None and not kw):

Lib/json/encoder.py

@@ -116,7 +116,7 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True,
 
         If check_circular is true, then lists, dicts, and custom encoded
         objects will be checked for circular references during encoding to
-        prevent an infinite recursion (which would cause an OverflowError).
+        prevent an infinite recursion (which would cause an RecursionError).
         Otherwise, no such check takes place.
 
         If allow_nan is true, then NaN, Infinity, and -Infinity will be

Lib/json/tool.py

@@ -13,6 +13,7 @@
 import argparse
 import json
 import sys
+from pathlib import Path
 
 
 def main():
@@ -25,31 +26,60 @@ def main():
                         help='a JSON file to be validated or pretty-printed',
                         default=sys.stdin)
     parser.add_argument('outfile', nargs='?',
-                        type=argparse.FileType('w', encoding="utf-8"),
+                        type=Path,
                         help='write the output of infile to outfile',
-                        default=sys.stdout)
+                        default=None)
     parser.add_argument('--sort-keys', action='store_true', default=False,
                         help='sort the output of dictionaries alphabetically by key')
+    parser.add_argument('--no-ensure-ascii', dest='ensure_ascii', action='store_false',
+                        help='disable escaping of non-ASCII characters')
     parser.add_argument('--json-lines', action='store_true', default=False,
-                        help='parse input using the jsonlines format')
+                        help='parse input using the JSON Lines format. '
+                        'Use with --no-indent or --compact to produce valid JSON Lines output.')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--indent', default=4, type=int,
+                       help='separate items with newlines and use this number '
+                       'of spaces for indentation')
+    group.add_argument('--tab', action='store_const', dest='indent',
+                       const='\t', help='separate items with newlines and use '
+                       'tabs for indentation')
+    group.add_argument('--no-indent', action='store_const', dest='indent',
+                       const=None,
+                       help='separate items with spaces rather than newlines')
+    group.add_argument('--compact', action='store_true',
+                       help='suppress all whitespace separation (most compact)')
     options = parser.parse_args()
 
-    infile = options.infile
-    outfile = options.outfile
-    sort_keys = options.sort_keys
-    json_lines = options.json_lines
-    with infile, outfile:
+    dump_args = {
+        'sort_keys': options.sort_keys,
+        'indent': options.indent,
+        'ensure_ascii': options.ensure_ascii,
+    }
+    if options.compact:
+        dump_args['indent'] = None
+        dump_args['separators'] = ',', ':'
+
+    with options.infile as infile:
         try:
-            if json_lines:
+            if options.json_lines:
                 objs = (json.loads(line) for line in infile)
             else:
-                objs = (json.load(infile), )
-            for obj in objs:
-                json.dump(obj, outfile, sort_keys=sort_keys, indent=4)
-                outfile.write('\n')
+                objs = (json.load(infile),)
+
+            if options.outfile is None:
+                out = sys.stdout
+            else:
+                out = options.outfile.open('w', encoding='utf-8')
+            with out as outfile:
+                for obj in objs:
+                    json.dump(obj, outfile, **dump_args)
+                    outfile.write('\n')
         except ValueError as e:
             raise SystemExit(e)
 
 
 if __name__ == '__main__':
-    main()
+    try:
+        main()
+    except BrokenPipeError as exc:
+        sys.exit(exc.errno)

Lib/test/test_json/__init__.py

@@ -6,6 +6,7 @@
 from test import support
 from test.support import import_helper
 
+
 # import json with and without accelerations
 # XXX RUSTPYTHON: we don't import _json as fresh since the fresh module isn't placed
 # into the sys.modules cache, and therefore the vm can't recognize the _json.Scanner class
@@ -40,7 +41,6 @@ def test_pyjson(self):
                          'json.encoder')
 
 class TestCTest(CTest):
-    @unittest.expectedFailure
     def test_cjson(self):
         self.assertEqual(self.json.scanner.make_scanner.__module__, '_json')
         self.assertEqual(self.json.decoder.scanstring.__module__, '_json')

Lib/test/test_json/test_decode.py

@@ -97,10 +97,6 @@ def test_negative_index(self):
         d = self.json.JSONDecoder()
         self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000)
 
-    def test_deprecated_encode(self):
-        with self.assertWarns(DeprecationWarning):
-            self.loads('{}', encoding='fake')
-
 class TestPyDecode(TestDecode, PyTest): pass
 # TODO: RUSTPYTHON
 class TestCDecode(TestDecode, CTest): # pass

Lib/test/test_json/test_recursion.py

@@ -1,3 +1,4 @@
+from test import support
 from test.test_json import PyTest, CTest
 
 
@@ -52,7 +53,7 @@ def default(self, o):
                         return [JSONTestObject]
                     else:
                         return 'JSONTestObject'
-                return pyjson.JSONEncoder.default(o)
+                return self.json.JSONEncoder.default(o)
 
         enc = RecursiveJSONEncoder()
         self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"')
@@ -69,21 +70,26 @@ def test_highly_nested_objects_decoding(self):
         # test that loading highly-nested objects doesn't segfault when C
         # accelerations are used. See #12017
         with self.assertRaises(RecursionError):
-            self.loads('{"a":' * 100000 + '1' + '}' * 100000)
+            with support.infinite_recursion():
+                self.loads('{"a":' * 100000 + '1' + '}' * 100000)
         with self.assertRaises(RecursionError):
-            self.loads('{"a":' * 100000 + '[1]' + '}' * 100000)
+            with support.infinite_recursion():
+                self.loads('{"a":' * 100000 + '[1]' + '}' * 100000)
         with self.assertRaises(RecursionError):
-            self.loads('[' * 100000 + '1' + ']' * 100000)
+            with support.infinite_recursion():
+                self.loads('[' * 100000 + '1' + ']' * 100000)
 
     def test_highly_nested_objects_encoding(self):
         # See #12051
         l, d = [], {}
         for x in range(100000):
             l, d = [l], {'k':d}
         with self.assertRaises(RecursionError):
-            self.dumps(l)
+            with support.infinite_recursion():
+                self.dumps(l)
         with self.assertRaises(RecursionError):
-            self.dumps(d)
+            with support.infinite_recursion():
+                self.dumps(d)
 
     def test_endless_recursion(self):
         # See #12051
@@ -93,7 +99,8 @@ def default(self, o):
                 return [o]
 
         with self.assertRaises(RecursionError):
-            EndlessJSONEncoder(check_circular=False).encode(5j)
+            with support.infinite_recursion():
+                EndlessJSONEncoder(check_circular=False).encode(5j)
 
 
 class TestPyRecursion(TestRecursion, PyTest): pass

Lib/test/test_json/test_speedups.py

@@ -62,6 +62,15 @@ def bad_encoder2(*args):
         with self.assertRaises(ZeroDivisionError):
             enc('spam', 4)
 
+    def test_bad_markers_argument_to_encoder(self):
+        # https://bugs.python.org/issue45269
+        with self.assertRaisesRegex(
+            TypeError,
+            r'make_encoder\(\) argument 1 must be dict or None, not int',
+        ):
+            self.json.encoder.c_make_encoder(1, None, None, None, ': ', ', ',
+                                             False, False, False)
+
     # TODO: RUSTPYTHON, translate the encoder to Rust
     @unittest.expectedFailure
     def test_bad_bool_args(self):

Lib/test/test_json/test_tool.py

@@ -1,8 +1,10 @@
+import errno
 import os
 import sys
 import textwrap
 import unittest
-from subprocess import Popen, PIPE
+import subprocess
+
 from test import support
 from test.support import os_helper
 from test.support.script_helper import assert_python_ok
@@ -85,10 +87,9 @@ class TestTool(unittest.TestCase):
 
     def test_stdin_stdout(self):
         args = sys.executable, '-m', 'json.tool'
-        with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
-            out, err = proc.communicate(self.data.encode())
-        self.assertEqual(out.splitlines(), self.expect.encode().splitlines())
-        self.assertEqual(err, b'')
+        process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True)
+        self.assertEqual(process.stdout, self.expect)
+        self.assertEqual(process.stderr, '')
 
     def _create_infile(self, data=None):
         infile = os_helper.TESTFN
@@ -124,18 +125,26 @@ def test_infile_outfile(self):
         outfile = os_helper.TESTFN + '.out'
         rc, out, err = assert_python_ok('-m', 'json.tool', infile, outfile)
         self.addCleanup(os.remove, outfile)
-        with open(outfile, "r") as fp:
+        with open(outfile, "r", encoding="utf-8") as fp:
+            self.assertEqual(fp.read(), self.expect)
+        self.assertEqual(rc, 0)
+        self.assertEqual(out, b'')
+        self.assertEqual(err, b'')
+
+    def test_writing_in_place(self):
+        infile = self._create_infile()
+        rc, out, err = assert_python_ok('-m', 'json.tool', infile, infile)
+        with open(infile, "r", encoding="utf-8") as fp:
             self.assertEqual(fp.read(), self.expect)
         self.assertEqual(rc, 0)
         self.assertEqual(out, b'')
         self.assertEqual(err, b'')
 
     def test_jsonlines(self):
         args = sys.executable, '-m', 'json.tool', '--json-lines'
-        with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
-            out, err = proc.communicate(self.jsonlines_raw.encode())
-        self.assertEqual(out.splitlines(), self.jsonlines_expect.encode().splitlines())
-        self.assertEqual(err, b'')
+        process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True)
+        self.assertEqual(process.stdout, self.jsonlines_expect)
+        self.assertEqual(process.stderr, '')
 
     def test_help_flag(self):
         rc, out, err = assert_python_ok('-m', 'json.tool', '-h')
@@ -150,3 +159,73 @@ def test_sort_keys_flag(self):
         self.assertEqual(out.splitlines(),
                          self.expect_without_sort_keys.encode().splitlines())
         self.assertEqual(err, b'')
+
+    def test_indent(self):
+        input_ = '[1, 2]'
+        expect = textwrap.dedent('''\
+        [
+          1,
+          2
+        ]
+        ''')
+        args = sys.executable, '-m', 'json.tool', '--indent', '2'
+        process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True)
+        self.assertEqual(process.stdout, expect)
+        self.assertEqual(process.stderr, '')
+
+    def test_no_indent(self):
+        input_ = '[1,\n2]'
+        expect = '[1, 2]\n'
+        args = sys.executable, '-m', 'json.tool', '--no-indent'
+        process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True)
+        self.assertEqual(process.stdout, expect)
+        self.assertEqual(process.stderr, '')
+
+    def test_tab(self):
+        input_ = '[1, 2]'
+        expect = '[\n\t1,\n\t2\n]\n'
+        args = sys.executable, '-m', 'json.tool', '--tab'
+        process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True)
+        self.assertEqual(process.stdout, expect)
+        self.assertEqual(process.stderr, '')
+
+    def test_compact(self):
+        input_ = '[ 1 ,\n 2]'
+        expect = '[1,2]\n'
+        args = sys.executable, '-m', 'json.tool', '--compact'
+        process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True)
+        self.assertEqual(process.stdout, expect)
+        self.assertEqual(process.stderr, '')
+
+    def test_no_ensure_ascii_flag(self):
+        infile = self._create_infile('{"key":"💩"}')
+        outfile = os_helper.TESTFN + '.out'
+        self.addCleanup(os.remove, outfile)
+        assert_python_ok('-m', 'json.tool', '--no-ensure-ascii', infile, outfile)
+        with open(outfile, "rb") as f:
+            lines = f.read().splitlines()
+        # asserting utf-8 encoded output file
+        expected = [b'{', b'    "key": "\xf0\x9f\x92\xa9"', b"}"]
+        self.assertEqual(lines, expected)
+
+    def test_ensure_ascii_default(self):
+        infile = self._create_infile('{"key":"💩"}')
+        outfile = os_helper.TESTFN + '.out'
+        self.addCleanup(os.remove, outfile)
+        assert_python_ok('-m', 'json.tool', infile, outfile)
+        with open(outfile, "rb") as f:
+            lines = f.read().splitlines()
+        # asserting an ascii encoded output file
+        expected = [b'{', rb'    "key": "\ud83d\udca9"', b"}"]
+        self.assertEqual(lines, expected)
+
+    @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows")
+    def test_broken_pipe_error(self):
+        cmd = [sys.executable, '-m', 'json.tool']
+        proc = subprocess.Popen(cmd,
+                                stdout=subprocess.PIPE,
+                                stdin=subprocess.PIPE)
+        # bpo-39828: Closing before json.tool attempts to write into stdout.
+        proc.stdout.close()
+        proc.communicate(b'"{}"')
+        self.assertEqual(proc.returncode, errno.EPIPE)