Merge pull request #2 from tranvietanh1991/develop

encode decode all datetime type and decimal, using ExtType. And fully…

.travis.yml

@@ -2,37 +2,38 @@ sudo: false
 language: python
 dist: xenial
 python:
-- '2.7'
-- '3.5'
-- '3.6'
-- '3.7'
+  - '2.7'
+  - '3.5'
+  - '3.6'
+  - '3.7'
 addons:
   apt_packages:
-  - libenchant-dev
+    - libenchant-dev
+
 install:
-- pip install tox-travis virtualenv tox python-coveralls coveralls
+  - pip install tox-travis virtualenv tox python-coveralls coveralls
 cache:
   directories:
-  - "$HOME/.cache/pip"
+    - "$HOME/.cache/pip"
 script:
-- QUIET=true tox
+  - QUIET=true tox
 stages:
-- test
-- deploy
+  - test
+  - deploy
 jobs:
   include:
-  - stage: test
-    after_success:
-    - coveralls
-  - stage: deploy
-    python: 2.7
-    script: skip
-    install: skip
-    if: repo = "tranvietanh1991/nameko-django"
-    deploy:
-      provider: pypi
-      user: tranvietanh1991
-      password:
-        secure: ml3I7E9idFy7Yjm6POQOjX4bwobNh23+QhgXfEgOywhF8icbLbB+eRYwVDflC9ekuBiJrWr1aPliXjpWphrGG+z/bawSntv/zERhCjlq3fPXYZnhnGiyZqrfwvq0j67niIBTRUGy51q2Nd9kF5hKivlsLS1XSsodUx2lW4CoeIadLIJIzq9bByVZS3eVGp3e9Nvh598TznWWOu9eKI3lRWEJPNqXQ10K4TmIfQ1y2MyEMw6l8azFKT9raFtKWZO2b92Ie6MXOs+Pf+BuVNZp83FGVxxj6txF43kZseDHyRECfxCj4WgCY78pFfgeBri1lFRX4rKrseMifEx+5YWWbMjn37a456lqxr6UBd15WKr0SHKnZy8eurlPYS8qWWtapnrYCm3W5UAPPsujVFZX25BTTkfcRh9RQxEXzhOXGnpyB1KFBmi/c362c7zOCmzfe3412wZgIBcCchKMrBjoolSxr1rVMUZb4L6I68LLY1Udb0B8Zagyw9GO28PsNm+Dfdgr4DmPn1svXztbWrhsud04C6xFQj9KAdyBN1kgig6kXDUgYmdoSKfUrYi50qzkZ+hwl1EszQy7+Tr7Gs6se0bC+9fYLzNeFiKWHN6YsfVp+4fS1+kTwroZ4m0r9Vo86ZCMO2kfxVI4x4j8t4C9bli1Xzs3GwtXsgwev0U5S14=
-      on:
-        branch: master
+    - stage: test
+      after_success:
+        - coveralls
+    - stage: deploy
+      python: 2.7
+      script: skip
+      install: skip
+      if: repo = "tranvietanh1991/nameko-django"
+      deploy:
+        provider: pypi
+        user: tranvietanh1991
+        password:
+          secure: ml3I7E9idFy7Yjm6POQOjX4bwobNh23+QhgXfEgOywhF8icbLbB+eRYwVDflC9ekuBiJrWr1aPliXjpWphrGG+z/bawSntv/zERhCjlq3fPXYZnhnGiyZqrfwvq0j67niIBTRUGy51q2Nd9kF5hKivlsLS1XSsodUx2lW4CoeIadLIJIzq9bByVZS3eVGp3e9Nvh598TznWWOu9eKI3lRWEJPNqXQ10K4TmIfQ1y2MyEMw6l8azFKT9raFtKWZO2b92Ie6MXOs+Pf+BuVNZp83FGVxxj6txF43kZseDHyRECfxCj4WgCY78pFfgeBri1lFRX4rKrseMifEx+5YWWbMjn37a456lqxr6UBd15WKr0SHKnZy8eurlPYS8qWWtapnrYCm3W5UAPPsujVFZX25BTTkfcRh9RQxEXzhOXGnpyB1KFBmi/c362c7zOCmzfe3412wZgIBcCchKMrBjoolSxr1rVMUZb4L6I68LLY1Udb0B8Zagyw9GO28PsNm+Dfdgr4DmPn1svXztbWrhsud04C6xFQj9KAdyBN1kgig6kXDUgYmdoSKfUrYi50qzkZ+hwl1EszQy7+Tr7Gs6se0bC+9fYLzNeFiKWHN6YsfVp+4fS1+kTwroZ4m0r9Vo86ZCMO2kfxVI4x4j8t4C9bli1Xzs3GwtXsgwev0U5S14=
+        on:
+          branch: master

README.md

@@ -38,3 +38,26 @@ SERIALIZERS:
 ```
 This will accept both of the `msgpack` and `django_msgpackpickle` but only output of result portfolio using `msgpack`
 Once all service migrated, then switch to the first configuration
+
+## Features
+### This serializer will automatically encode and decode: 
+- DateTime, Date, Time, Duration: 
+    object will be converted to string representation compatible with django.utils.dateparse 
+    and convert back using django.utils.dateparse()
+- Decimal:
+    object will be converted to byte string and then recover back to Decimal
+- Django ORM instance:
+    object will be pickled using python cPickle/pickle library and depickled back to ORM Model instance
+- Django ORM queryset:
+    object will be deform to Model + Query then pickled to avoid sending a list of instance
+
+### String evaluation
+This serializer can evaluate string that is compatible with `django.utils.dateparse` format 
+and auto convert the string to either `DateTime`, `Date`, `Time`, `Duration` object.
+
+Also it can evaluate string with format like this:
+`"<app_name.model_name.ID>"`  this will be converted to an ORM instance: using `Model.objects.get(pk=ID)`
+For example: `<auth.User.1>`
+
+`"(app_name.model_name: RAW_QUERY_WITHOUT_SELECT_FROM)"` this will be converted to an ORM queryset
+For example: `(auth.User: id >= 1 and date_joined > '2018-11-22 00:47:14.263837')`

nameko_django/VERSION

@@ -1 +1 @@
-1.0.1
+1.1.0

nameko_django/serializer.py

@@ -9,136 +9,173 @@
 #
 from __future__ import unicode_literals
 
-import os
-from datetime import datetime
-from decimal import Decimal
-from io import BytesIO
-
-from aenum import Enum
+from datetime import datetime, date, time, timedelta
+from decimal import Decimal, ROUND_HALF_EVEN, Context, Overflow, DivisionByZero, InvalidOperation
+from aenum import Enum, IntEnum, Constant
 from django.db.models import Model, QuerySet
 from django.db.models.base import ModelBase
 from django.db.models.sql.query import Query
 from django.utils import dateparse
-from msgpack import packb, unpackb
-from six import string_types
+from msgpack import packb, unpackb, ExtType
+from six import string_types, ensure_str, ensure_binary
+import re
 
 try:
     import cPickle as pickle
 except ImportError:
     import pickle
 
-DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT = os.getenv("DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT", "%Y-%m-%d %H:%M:%S.%f%z")
+import logging
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_DATE_STRING_FORMAT = "%Y-%m-%d"
+DEFAULT_TIME_STRING_FORMAT = "%H:%M:%S.%f"
+DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT = "%Y-%m-%d %H:%M:%S.%f%z"
+DEFAULT_DECIMAL_CONTEXT = Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999,
+                                  capitals=1, flags=[], traps=[Overflow, DivisionByZero,
+                                                               InvalidOperation])
+
+
+def pack(s):
+    return packb(s, use_bin_type=True)
+
+
+def unpack(s):
+    return unpackb(s, raw=False)
+
+
+class ExternalType(IntEnum):
+    DECIMAL = 42
+    ORM_INSTANCE = 43
+    ORM_QUERYSET = 44
 
 
-def serializable(obj):
-    """ Make an object serializable for JSON, msgpack
+def encode_nondefault_object(obj):
+    """ Encode an object by make it compatible with default msgpack encoder or using ExtType
 
-    :param obj: Namedtuple instance
+    :param obj: any objet
     :return:
     """
     if obj is None:
         return
     if hasattr(obj, '_asdict') and callable(obj._asdict):
-        result_obj = dict(obj._asdict())
+        return dict(obj._asdict())
     elif isinstance(obj, Enum) and hasattr(obj, 'value'):
         return obj.value
+    elif isinstance(obj, Constant) and hasattr(obj, '_value_'):
+        return obj._value_
     elif isinstance(obj, Decimal):
-        return float(obj)
+        return ExtType(ExternalType.DECIMAL, ensure_binary(str(obj)))
     elif isinstance(obj, datetime):
         return obj.strftime(DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT)
+    elif isinstance(obj, date):
+        return obj.strftime(DEFAULT_DATE_STRING_FORMAT)
+    elif isinstance(obj, time):
+        return obj.strftime(DEFAULT_TIME_STRING_FORMAT)
+    elif isinstance(obj, timedelta):
+        if 0 <= obj.total_seconds() < 86400:
+            return '+{}'.format(obj)
+        return str(obj)
     else:
-        dump_obj = django_pickle_dumps(obj)
-        if dump_obj is not None:
-            return dump_obj
-        else:
-            result_obj = obj
-
-    if isinstance(result_obj, dict):
-        return {
-            key: serializable(value)
-            for key, value in result_obj.items()
-        }
-    elif isinstance(result_obj, list) or isinstance(result_obj, set) or isinstance(result_obj, tuple):
-        return [serializable(value) for value in result_obj]
-    else:
-        return result_obj
-
-
-def django_is_pickable(s):
-    if isinstance(s, string_types) and len(s) > 255 and s[:2] == b'\x80\x02' and s[-1] == b'.':
-        return True
-    return False
-
-
-def django_pickle_dumps(obj):
-    if isinstance(obj, Model):
-        return pickle.dumps(obj, -1)
-    elif isinstance(obj, QuerySet):
-        return pickle.dumps((obj.model, obj.query), -1)
-    else:
-        return None
-
-
-def django_pickle_loads(obj_string):
-    objs = pickle.loads(obj_string)
-    if isinstance(objs, tuple) and len(objs) == 2:
+        if isinstance(obj, Model):
+            return ExtType(ExternalType.ORM_INSTANCE, pickle.dumps(obj, -1))
+        elif isinstance(obj, QuerySet):
+            return ExtType(ExternalType.ORM_QUERYSET, pickle.dumps((obj.model, obj.query), -1))
+    logger.debug("unknown type obj=%s", obj)
+    return obj
+
+
+def django_ext_hook(code, data):
+    if code == ExternalType.DECIMAL:
+        return Decimal(ensure_str(data, encoding='utf-8'), context=DEFAULT_DECIMAL_CONTEXT)
+    elif code == ExternalType.ORM_INSTANCE:
+        return pickle.loads(data)
+    elif code == ExternalType.ORM_QUERYSET:
         # untouched queryset case
-        model, query = objs
+        model, query = pickle.loads(data)
         if isinstance(model, ModelBase) and isinstance(query, Query):
             qs = model.objects.all()
             qs.query = query
             return qs
-    # normal case
-    return objs
+    # unable to decode external type then return as it is
+    return ExtType(code, data)
 
 
-def deserializable(obj):
-    """ Make an object serializable for JSON, msgpack
+def decode_dict_object(dict_obj):
+    logger.debug("decode dict obj=%s", dict_obj)
+    return {
+        key: decode_single_object(value)
+        for key, value in dict_obj.items()
+    }
 
-    :param obj: Namedtuple instance
-    :return:
-    """
-    if obj is None:
-        return
-    if isinstance(obj, string_types):
-        if django_is_pickable(obj):
-            dump_obj = django_pickle_loads(obj)
-        else:
-            dump_obj = dateparse.parse_datetime(obj)
-        if dump_obj is not None:
-            return dump_obj
-        else:
-            result_obj = obj
-    else:
-        result_obj = obj
-
-    if isinstance(result_obj, dict):
-        return {
-            key: deserializable(value)
-            for key, value in result_obj.items()
-        }
-    elif isinstance(result_obj, list) or isinstance(result_obj, set) or isinstance(result_obj, tuple):
-        return [deserializable(value) for value in result_obj]
-    else:
-        return result_obj
 
+def decode_list_object(list_obj):
+    logger.debug("decode list obj=%s", list_obj)
+    return [decode_single_object(value) for value in list_obj]
 
-def pack(s):
-    return packb(s, use_bin_type=True)
 
+datetime_test_re = re.compile(
+    r'[-+.:0123456789]*:[-+.:0123456789]+'  # datetime
+    r'|\d+\-\d+\-\d+'  # date
+    r'|[-+]?\d+\s+days?,?\s*[.:0123456789]*'  # duration
+    r'|[-+]?P\d*D?T\d*H?\d*M?\d*S?'  # duration ISO_8601
+)
 
-def unpack(s):
-    return unpackb(s, raw=False)
+django_orm_re = re.compile(r"<([\w]+\.[\w]+)\.(\d+)>")
+django_orm_queryset_re = re.compile(r"\s*\(([\w]+\.[\w]+):\s*(.+)\)", re.MULTILINE)
+
+from django.apps import apps
+
+
+def decode_single_object(obj):
+    if obj is None:
+        return
+    logger.debug("decode single obj=%s", obj)
+    if isinstance(obj, string_types):
+        datetime_obj = None
+        lenobj = len(obj)
+        if lenobj <= 33 and datetime_test_re.match(obj):
+            if lenobj == 33:
+                datetime_obj = dateparse.parse_datetime(obj.replace(' +', '+'))
+            elif 31 <= lenobj <= 32 or 21 <= lenobj <= 26:
+                datetime_obj = dateparse.parse_datetime(obj)
+            elif lenobj == 10:
+                datetime_obj = dateparse.parse_date(obj)
+                if not datetime_obj:  # there is an over lapse case
+                    datetime_obj = dateparse.parse_time(obj)
+            elif lenobj == 5 or lenobj == 8 or 10 <= lenobj <= 15:
+                datetime_obj = dateparse.parse_time(obj)
+            if datetime_obj is None:  # a time object is also maybe a valid duration object
+                datetime_obj = dateparse.parse_duration(re.sub(r'^(\-?)\+?:?(\d)', r'\1\2', obj))
+        # if there is a datetime_obj can be decoded from string then return it
+        if datetime_obj is not None:
+            return datetime_obj
+        # check django orm evaluation from string
+        m = django_orm_re.match(obj)
+        if m:
+            return apps.get_model(m.group(1)).objects.get(pk=m.group(2))
+        m2 = django_orm_queryset_re.match(obj)
+        if m2:
+            model = apps.get_model(m2.group(1))
+            raw_id_qs = "SELECT id FROM {} WHERE {}".format(model._meta.db_table, m2.group(2).strip())
+            return model.objects.filter(id__in=[o.id for o in model.objects.raw(raw_id_qs)])
+    return obj
 
 
 def dumps(o):
-    return pack(serializable(o))
+    # logger.debug("dumps obj=%s", o)
+    return packb(o, strict_types=True, default=encode_nondefault_object, use_bin_type=True)
 
 
 def loads(s):
     if not isinstance(s, string_types):
-        s = BytesIO(s)
-    return deserializable(unpack(s))
+        s = bytes(s)
+    r = unpackb(s, ext_hook=django_ext_hook, object_hook=decode_dict_object, list_hook=decode_list_object, raw=False)
+    if isinstance(r, string_types):
+        return decode_single_object(r)
+    else:
+        return r
 
 
 register_args = (dumps, loads, 'application/x-django-msgpackpickle', 'binary')

requirements.txt

@@ -1,5 +1,10 @@
 django==1.11.*
 nameko==2.11.*
 msgpack==0.5.*
+aenum==2.1.*
 tox==3.13.*
-flake8
+flake8==3.7.8
+nose==1.3.7
+pytest==4.6.6
+pytest-django==3.6.0
+pytest-runner==5.1