Use decimal from str (#1518)

add use_decimal_from_str feature to JSON object serializer

.github/workflows/main.yml

@@ -62,7 +62,7 @@ jobs:
 
         # https://github.com/actions/setup-python#specifying-a-pypy-version
         python-version: ['3.7', '3.10', 'pypy-3.8']
-        framework: ['asyncio', 'tw203', 'twtrunk']
+        framework: ['asyncio', 'tw1910', 'tw221', 'twtrunk']
 
     # https://github.blog/changelog/2020-04-15-github-actions-new-workflow-features/
     # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepscontinue-on-error

Makefile

@@ -41,7 +41,7 @@ upload: clean
 		s3://fabric-deploy/autobahn/
 
 # cleanup everything
-clean:
+clean: clean_docs
 	-rm -f ./*.so
 	-rm -rf ./docs/build
 	-rm -rf ./.cache
@@ -73,7 +73,7 @@ publish: clean
 	twine upload dist/*
 
 clean_docs:
-	-rm -rf ./docs/build
+	-rm -rf ./docs/_build
 
 docs:
 	tox -e sphinx
@@ -175,8 +175,8 @@ test_util:
 test_rng:
 	USE_TWISTED=1 trial autobahn.test.test_rng
 
-test_tx_serializer:
-	USE_TWISTED=1 trial autobahn.wamp.test.test_serializer
+test_serializer:
+	USE_TWISTED=1 trial autobahn.wamp.test.test_wamp_serializer
 
 test_tx_cryptobox:
 	USE_TWISTED=1 trial autobahn.wamp.test.test_cryptobox

README.rst

@@ -249,19 +249,6 @@ Further, to speed up JSON on CPython using ``ujson``, set the environment variab
     This ability depends on features of the regular JSON standard library module
     not available on ``ujson``.
 
-To use ``cbor2``, an alternative, highly flexible and standards complicant CBOR
-implementation, set the environment variable:
-
-.. code:: console
-
-    AUTOBAHN_USE_CBOR2=1
-
-.. note::
-
-    ``cbor2`` is not used by default, because it is significantly slower currently
-    in our benchmarking for WAMP message serialization on both CPython and PyPy
-    compared to ``cbor``.
-
 
 .. |Version| image:: https://img.shields.io/pypi/v/autobahn.svg
    :target: https://pypi.python.org/pypi/autobahn

autobahn/_version.py

@@ -24,6 +24,6 @@
 #
 ###############################################################################
 
-__version__ = '22.1.1'
+__version__ = '22.2.1.dev1'
 
 __build__ = u'00000000-0000000'

autobahn/wamp/message.py

@@ -33,7 +33,7 @@
 from autobahn.wamp.role import ROLE_NAME_TO_CLASS
 
 try:
-    import cbor
+    import cbor2
     import flatbuffers
     from autobahn.wamp import message_fbs
 except ImportError:
@@ -1987,7 +1987,7 @@ def topic(self, value):
     def args(self):
         if self._args is None and self._from_fbs:
             if self._from_fbs.ArgsLength():
-                self._args = cbor.loads(bytes(self._from_fbs.ArgsAsBytes()))
+                self._args = cbor2.loads(bytes(self._from_fbs.ArgsAsBytes()))
         return self._args
 
     @args.setter
@@ -1999,7 +1999,7 @@ def args(self, value):
     def kwargs(self):
         if self._kwargs is None and self._from_fbs:
             if self._from_fbs.KwargsLength():
-                self._kwargs = cbor.loads(bytes(self._from_fbs.KwargsAsBytes()))
+                self._kwargs = cbor2.loads(bytes(self._from_fbs.KwargsAsBytes()))
         return self._kwargs
 
     @kwargs.setter
@@ -2222,11 +2222,11 @@ def build(self, builder):
 
         args = self.args
         if args:
-            args = builder.CreateByteVector(cbor.dumps(args))
+            args = builder.CreateByteVector(cbor2.dumps(args))
 
         kwargs = self.kwargs
         if kwargs:
-            kwargs = builder.CreateByteVector(cbor.dumps(kwargs))
+            kwargs = builder.CreateByteVector(cbor2.dumps(kwargs))
 
         payload = self.payload
         if payload:
@@ -3369,7 +3369,7 @@ def publication(self, value):
     def args(self):
         if self._args is None and self._from_fbs:
             if self._from_fbs.ArgsLength():
-                self._args = cbor.loads(bytes(self._from_fbs.ArgsAsBytes()))
+                self._args = cbor2.loads(bytes(self._from_fbs.ArgsAsBytes()))
         return self._args
 
     @args.setter
@@ -3381,7 +3381,7 @@ def args(self, value):
     def kwargs(self):
         if self._kwargs is None and self._from_fbs:
             if self._from_fbs.KwargsLength():
-                self._kwargs = cbor.loads(bytes(self._from_fbs.KwargsAsBytes()))
+                self._kwargs = cbor2.loads(bytes(self._from_fbs.KwargsAsBytes()))
         return self._kwargs
 
     @kwargs.setter
@@ -3533,11 +3533,11 @@ def build(self, builder):
 
         args = self.args
         if args:
-            args = builder.CreateByteVector(cbor.dumps(args))
+            args = builder.CreateByteVector(cbor2.dumps(args))
 
         kwargs = self.kwargs
         if kwargs:
-            kwargs = builder.CreateByteVector(cbor.dumps(kwargs))
+            kwargs = builder.CreateByteVector(cbor2.dumps(kwargs))
 
         payload = self.payload
         if payload:

autobahn/wamp/serializer.py

@@ -25,9 +25,11 @@
 ###############################################################################
 
 import os
+import re
 import struct
 import platform
 import math
+import decimal
 from binascii import b2a_hex, a2b_hex
 
 from txaio import time_ns
@@ -292,8 +294,6 @@ def unserialize(self, payload, isBinary=None):
                 message_type = raw_msg[0]
 
                 if type(message_type) != int:
-                    # CBOR doesn't roundtrip number types
-                    # https://bitbucket.org/bodhisnarkva/cbor/issues/6/number-types-dont-roundtrip
                     raise ProtocolError("invalid type {0} for WAMP message type".format(type(message_type)))
 
                 Klass = self.MESSAGE_TYPE_MAP.get(message_type)
@@ -358,6 +358,8 @@ def default(self, obj):
                     return '0x' + b2a_hex(obj).decode('ascii')
                 else:
                     return '\x00' + base64.b64encode(obj).decode('ascii')
+            elif isinstance(obj, decimal.Decimal):
+                return str(obj)
             else:
                 return json.JSONEncoder.default(self, obj)
 
@@ -368,6 +370,8 @@ def default(self, obj):
     from json import scanner
     from json.decoder import scanstring
 
+    _DEC_MATCH = re.compile(r'^[\+\-E\.0-9]+$')
+
     class _WAMPJsonDecoder(json.JSONDecoder):
 
         def __init__(self, *args, **kwargs):
@@ -377,16 +381,30 @@ def __init__(self, *args, **kwargs):
             else:
                 self._use_binary_hex_encoding = False
 
+            if 'use_decimal_from_str' in kwargs:
+                self._use_decimal_from_str = kwargs['use_decimal_from_str']
+                del kwargs['use_decimal_from_str']
+            else:
+                self._use_decimal_from_str = False
+
             json.JSONDecoder.__init__(self, *args, **kwargs)
 
             def _parse_string(*args, **kwargs):
                 s, idx = scanstring(*args, **kwargs)
                 if self._use_binary_hex_encoding:
                     if s and s[0:2] == '0x':
                         s = a2b_hex(s[2:])
+                        return s, idx
                 else:
                     if s and s[0] == '\x00':
                         s = base64.b64decode(s[1:])
+                        return s, idx
+                if self._use_decimal_from_str and _DEC_MATCH.match(s):
+                    try:
+                        s = decimal.Decimal(s)
+                        return s, idx
+                    except decimal.InvalidOperation:
+                        pass
                 return s, idx
 
             self.parse_string = _parse_string
@@ -398,9 +416,10 @@ def _parse_string(*args, **kwargs):
             # not the C version, as the latter won't work
             # self.scan_once = scanner.make_scanner(self)
 
-    def _loads(s, use_binary_hex_encoding=False):
+    def _loads(s, use_binary_hex_encoding=False, use_decimal_from_str=False):
         return json.loads(s,
                           use_binary_hex_encoding=use_binary_hex_encoding,
+                          use_decimal_from_str=use_decimal_from_str,
                           cls=_WAMPJsonDecoder)
 
     def _dumps(obj, use_binary_hex_encoding=False):
@@ -425,14 +444,23 @@ class JsonObjectSerializer(object):
 
     BINARY = False
 
-    def __init__(self, batched=False, use_binary_hex_encoding=False):
+    def __init__(self, batched=False, use_binary_hex_encoding=False, use_decimal_from_str=False):
         """
 
         :param batched: Flag that controls whether serializer operates in batched mode.
         :type batched: bool
+
+        :param use_binary_hex_encoding: Flag to enable HEX encoding prefixed with ``"0x"``,
+            otherwise prefix binaries with a ``\0`` byte.
+        :type use_binary_hex_encoding: bool
+
+        :param use_decimal_from_str: Flag to automatically encode Decimals as strings, and
+            to try to parse strings as Decimals.
+        :type use_decimal_from_str: bool
         """
         self._batched = batched
         self._use_binary_hex_encoding = use_binary_hex_encoding
+        self._use_decimal_from_str = use_decimal_from_str
 
     def serialize(self, obj):
         """
@@ -456,7 +484,9 @@ def unserialize(self, payload):
             chunks = [payload]
         if len(chunks) == 0:
             raise Exception("batch format error")
-        return [_loads(data.decode('utf8'), use_binary_hex_encoding=self._use_binary_hex_encoding) for data in chunks]
+        return [_loads(data.decode('utf8'),
+                       use_binary_hex_encoding=self._use_binary_hex_encoding,
+                       use_decimal_from_str=self._use_decimal_from_str) for data in chunks]
 
 
 IObjectSerializer.register(JsonObjectSerializer)
@@ -483,14 +513,16 @@ class JsonSerializer(Serializer):
     WAMP-over-Longpoll HTTP fallback.
     """
 
-    def __init__(self, batched=False):
+    def __init__(self, batched=False, use_binary_hex_encoding=False, use_decimal_from_str=False):
         """
         Ctor.
 
         :param batched: Flag to control whether to put this serialized into batched mode.
         :type batched: bool
         """
-        Serializer.__init__(self, JsonObjectSerializer(batched=batched))
+        Serializer.__init__(self, JsonObjectSerializer(batched=batched,
+                                                       use_binary_hex_encoding=use_binary_hex_encoding,
+                                                       use_decimal_from_str=use_decimal_from_str))
         if batched:
             self.SERIALIZER_ID = "json.batched"
 
@@ -640,32 +672,17 @@ def __init__(self, batched=False):
 
 
 _HAS_CBOR = False
-if 'AUTOBAHN_USE_CBOR2' in os.environ:
-    try:
-        # https://pypi.org/project/cbor2/
-        # https://github.com/agronholm/cbor2
-        import cbor2
-    except ImportError:
-        pass
-    else:
-        _HAS_CBOR = True
-        _cbor_loads = cbor2.loads
-        _cbor_dumps = cbor2.dumps
-        _cbor = cbor2
-        # print('Notice: Autobahn is using cbor2 library for CBOR serialization')
+
+
+try:
+    import cbor2
+except ImportError:
+    pass
 else:
-    try:
-        # https://pypi.python.org/pypi/cbor
-        # https://bitbucket.org/bodhisnarkva/cbor
-        import cbor
-    except ImportError:
-        pass
-    else:
-        _HAS_CBOR = True
-        _cbor_loads = cbor.loads
-        _cbor_dumps = cbor.dumps
-        _cbor = cbor
-        # print('Notice: Autobahn is using cbor library for CBOR serialization')
+    _HAS_CBOR = True
+    _cbor_loads = cbor2.loads
+    _cbor_dumps = cbor2.dumps
+    _cbor = cbor2
 
 
 if _HAS_CBOR: