Replace simplejson with rapidjson

src/client.py

@@ -22,7 +22,7 @@
 
 #
 # Copyright (c) 2007, 2019, Oracle and/or its affiliates. All rights reserved.
-# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -67,7 +67,6 @@
         import getopt
         import gettext
         import itertools
-        import simplejson as json
         import locale
         import logging
         import os
@@ -96,6 +95,7 @@
         import pkg.client.transport.transport as transport
         import pkg.client.options as options
         import pkg.fmri as fmri
+        import pkg.json as json
         import pkg.misc as misc
         import pkg.pipeutils as pipeutils
         import pkg.portable as portable

src/depot-config.py

@@ -22,6 +22,7 @@
 
 #
 # Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 import pkg.no_site_packages
@@ -34,7 +35,6 @@
 import re
 import shutil
 import six
-import simplejson as json
 import socket
 import sys
 import traceback
@@ -48,6 +48,7 @@
 import pkg.client.api_errors as apx
 import pkg.catalog
 import pkg.config as cfg
+import pkg.json as json
 import pkg.misc as misc
 import pkg.portable as portable
 import pkg.p5i as p5i

src/modules/catalog.py

@@ -20,6 +20,7 @@
 # CDDL HEADER END
 #
 # Copyright (c) 2007, 2016, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 
 """Interfaces and implementation for the Catalog object, as well as functions
 that operate on lists of package FMRIs."""
@@ -33,7 +34,6 @@
 import fnmatch
 import hashlib
 import os
-import simplejson as json
 import six
 import stat
 import threading
@@ -46,6 +46,7 @@
 import pkg.client.api_errors as api_errors
 import pkg.client.pkgdefs as pkgdefs
 import pkg.fmri as fmri
+import pkg.json as json
 import pkg.misc as misc
 import pkg.portable as portable
 import pkg.version
@@ -122,14 +123,12 @@ def signatures(self):
                 return { "sha-1": self.__sha_1_value }
 
         def _dump(self, obj, fp, skipkeys=False, ensure_ascii=True,
-            check_circular=True, allow_nan=True, cls=json.JSONEncoder,
-            indent=None, separators=None, encoding='utf-8', default=None, **kw):
-                iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
-                    check_circular=check_circular, allow_nan=allow_nan,
-                    indent=indent, separators=separators, encoding=encoding,
-                    default=default, **kw).iterencode(obj,
-                    _one_shot=self.__single_pass)
-                fp.writelines(misc.force_bytes(i) for i in iterable)
+            allow_nan=True, indent=None, default=None, **kw):
+
+                json.dump(obj=obj, stream=fp, skipkeys=skipkeys,
+                    ensure_ascii=ensure_ascii, allow_nan=allow_nan,
+                    indent=indent, default=default, chunk_size=self.__bufsz,
+                    **kw)
 
         def save(self):
                 """Serializes and stores the provided data in JSON format."""
@@ -146,8 +145,7 @@ def save(self):
                 if not out:
                         out = self
 
-                self._dump(self.__data, out, check_circular=False,
-                    separators=(",", ":"), sort_keys=self.__sign)
+                self._dump(self.__data, out, sort_keys=self.__sign)
                 out.write(b"\n")
 
                 if self.__fileobj:
@@ -183,8 +181,7 @@ def save(self):
                                 # Catalog is not empty, so a separator is needed.
                                 sfile.write(b",")
                         sfile.write(b'"_SIGNATURE":')
-                        self._dump(self.signatures(), sfile, check_circular=False,
-                            separators=(",", ":"))
+                        self._dump(self.signatures(), sfile)
                         sfile.write(b"}\n")
 
         def write(self, data):

src/modules/client/api.py

@@ -22,7 +22,7 @@
 
 #
 # Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
-# Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """This module provides the supported, documented interface for clients to
@@ -63,7 +63,6 @@
 import glob
 import os
 import shutil
-import simplejson as json
 import sys
 import tempfile
 import threading
@@ -90,6 +89,7 @@
 import pkg.client.publisher as publisher
 import pkg.client.query_parser as query_p
 import pkg.fmri as fmri
+import pkg.json as json
 import pkg.mediator as med
 import pkg.misc as misc
 import pkg.nrlock
@@ -1599,7 +1599,7 @@ def __plan_op(self, _op, _act_timeout=0, _ad_kwargs=None,
                         pd_json1 = self.__plan_desc.getstate(self.__plan_desc,
                             reset_volatiles=True)
                         fobj = tempfile.TemporaryFile(mode="w+")
-                        json.dump(pd_json1, fobj, encoding="utf-8")
+                        json.dump(pd_json1, fobj)
                         pd_new = plandesc.PlanDescription(_op)
                         pd_new._load(fobj)
                         pd_json2 = pd_new.getstate(pd_new, reset_volatiles=True)

src/modules/client/client_api.py

@@ -22,6 +22,7 @@
 
 #
 # Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 
@@ -31,7 +32,6 @@
 import errno
 import getopt
 import itertools
-import simplejson as json
 import os
 import re
 import six
@@ -41,7 +41,6 @@
 import textwrap
 import time
 import traceback
-import jsonschema
 
 from six.moves import filter, map, range
 
@@ -55,6 +54,7 @@
 import pkg.client.publisher as publisher
 import pkg.client.options as options
 import pkg.fmri as fmri
+import pkg.json as json
 import pkg.misc as misc
 import pkg.pipeutils as pipeutils
 import pkg.portable as portable
@@ -540,8 +540,8 @@ def __prepare_json(status, op=None, schema=None, data=None, errors=None):
         if op:
                 op_schema = _get_pkg_output_schema(op)
                 try:
-                        jsonschema.validate(ret_json, op_schema)
-                except jsonschema.ValidationError as e:
+                        json.validate(ret_json, op_schema)
+                except json.ValidationError as e:
                         newret_json = {"status": EXIT_OOPS,
                             "errors": [{"reason": str(e)}]}
                         return newret_json
@@ -2898,9 +2898,9 @@ def __pkg(subcommand, pargs_json, opts_json, pkg_image=None,
                 # Validate JSON input with JSON schema.
                 input_schema = _get_pkg_input_schema(subcommand,
                     opts_mapping=opts_mapping)
-                jsonschema.validate({arg_name: pargs, "opts_json": opts},
+                json.validate({arg_name: pargs, "opts_json": opts},
                     input_schema)
-        except jsonschema.ValidationError as e:
+        except json.ValidationError as e:
                 return None, __prepare_json(EXIT_BADOPT,
                     errors=[{"reason": str(e)}])
 

src/modules/client/image.py

@@ -22,7 +22,7 @@
 
 #
 # Copyright (c) 2007, 2019, Oracle and/or its affiliates. All rights reserved.
-# Copyright 2017 OmniOS Community Edition (OmniOSce) Association.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 import atexit
@@ -35,7 +35,6 @@
 import os
 import platform
 import shutil
-import simplejson as json
 import six
 import stat
 import sys
@@ -65,6 +64,7 @@
 import pkg.config                       as cfg
 import pkg.file_layout.layout           as fl
 import pkg.fmri
+import pkg.json                         as json
 import pkg.lockfile                     as lockfile
 import pkg.manifest                     as manifest
 import pkg.mediator                     as med
@@ -4805,7 +4805,7 @@ def ipkg_is_up_to_date(self, check_cancel, noexecute,
 
                 return img.imageplan.nothingtodo()
 
-        # avoid set implementation uses simplejson to store a set of pkg_stems
+        # avoid set implementation uses json to store a set of pkg_stems
         # being avoided (explicitly or implicitly), and a set of tracked stems
         # that are obsolete.
         #
@@ -4904,7 +4904,7 @@ def _avoid_set_save(self, new_set=None, implicit_avoid=None,
 
                 self.__avoid_set_altered = False
 
-        # frozen dict implementation uses simplejson to store a dictionary of
+        # frozen dict implementation uses json to store a dictionary of
         # pkg_stems that are frozen, the versions at which they're frozen, and
         # the reason, if given, why the package was frozen.
         #

src/modules/client/linkedimage/common.py

@@ -22,6 +22,7 @@
 
 #
 # Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """
@@ -47,7 +48,6 @@
 import operator
 import os
 import select
-import simplejson as json
 
 # Redefining built-in 'reduce', 'zip'; pylint: disable=W0622
 # Imports from package six are not grouped: pylint: disable=C0412
@@ -69,6 +69,7 @@
 import pkg.client.progress as progress
 import pkg.facet
 import pkg.fmri
+import pkg.json as json
 import pkg.misc as misc
 import pkg.pkgsubprocess
 import pkg.version
@@ -3603,6 +3604,21 @@ def get_inheritable_facets(img, pd=None):
 def save_data(path, data, root="/", catch_exception=True):
         """Save JSON encoded linked image metadata to a file."""
 
+        def PkgEncode(obj):
+                """Required routine that overrides the default base
+                class version.  This routine must serialize 'obj' when
+                attempting to save 'obj' json format."""
+
+                if isinstance(obj, (pkg.fmri.PkgFmri,
+                    pkg.client.linkedimage.common.LinkedImageName)):
+                        return str(obj)
+
+                if isinstance(obj, pkgplan.PkgPlan):
+                        return obj.getstate()
+
+                if isinstance(obj, (set, frozenset)):
+                        return list(obj)
+
         # make sure the directory we're about to save data into exists.
         path_dir = os.path.dirname(path)
         pathtmp = "{0}.{1:d}.tmp".format(path, os.getpid())
@@ -3616,8 +3632,7 @@ def save_data(path, data, root="/", catch_exception=True):
                 fd = ar.ar_open(root, pathtmp, os.O_WRONLY,
                     mode=0o644, create=True, truncate=True)
                 fobj = os.fdopen(fd, "w")
-                json.dump(data, fobj, encoding="utf-8",
-                    cls=pkg.client.linkedimage.PkgEncoder)
+                json.dump(data, fobj, default=PkgEncode)
                 fobj.close()
 
                 # atomically create the desired file
@@ -3643,8 +3658,7 @@ def load_data(path, missing_ok=False, root="/", decode=True,
 
                 fd = ar.ar_open(root, path, os.O_RDONLY)
                 fobj = os.fdopen(fd, "r")
-                data = json.load(fobj, encoding="utf-8",
-                    object_hook=object_hook)
+                data = json.load(fobj, object_hook=object_hook)
                 fobj.close()
         except OSError as e:
                 # W0212 Access to a protected member
@@ -3654,30 +3668,6 @@ def load_data(path, missing_ok=False, root="/", decode=True,
                 raise apx._convert_error(e)
         return data
 
-
-class PkgEncoder(json.JSONEncoder):
-        """Utility class used when json encoding linked image metadata."""
-
-        # E0202 An attribute inherited from JSONEncoder hide this method
-        # pylint: disable=E0202
-        def default(self, obj):
-                """Required routine that overrides the default base
-                class version.  This routine must serialize 'obj' when
-                attempting to save 'obj' json format."""
-
-                if isinstance(obj, (pkg.fmri.PkgFmri,
-                    pkg.client.linkedimage.common.LinkedImageName)):
-                        return str(obj)
-
-                if isinstance(obj, pkgplan.PkgPlan):
-                        return obj.getstate()
-
-                if isinstance(obj, (set, frozenset)):
-                        return list(obj)
-
-                return json.JSONEncoder.default(self, obj)
-
-
 def PkgDecoder(dct):
         """Utility class used when json decoding linked image metadata."""
         # Replace unicode keys/values with strings

src/modules/client/plandesc.py

@@ -22,6 +22,7 @@
 
 #
 # Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """
@@ -40,7 +41,6 @@
 import collections
 import itertools
 import operator
-import simplejson as json
 import six
 
 import pkg.actions
@@ -50,6 +50,7 @@
 import pkg.client.pkgplan
 import pkg.facet
 import pkg.fmri
+import pkg.json as json
 import pkg.misc
 import pkg.version
 
@@ -326,7 +327,7 @@ def _save(self, fobj, reset_volatiles=False):
                     reset_volatiles=reset_volatiles)
                 try:
                         fobj.truncate()
-                        json.dump(state, fobj, encoding="utf-8")
+                        json.dump(state, fobj)
                         fobj.flush()
                 except OSError as e:
                         # Access to protected member; pylint: disable=W0212
@@ -342,8 +343,7 @@ def _load(self, fobj):
 
                 try:
                         fobj.seek(0)
-                        state = json.load(fobj, encoding="utf-8",
-                            object_hook=pkg.misc.json_hook)
+                        state = json.load(fobj, object_hook=pkg.misc.json_hook)
                 except OSError as e:
                         # Access to protected member; pylint: disable=W0212
                         raise apx._convert_error(e)