diff --git a/.gitignore b/.gitignore index 894a44c..c26a45c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,72 @@ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/ +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties +### macOS template +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..64084b7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,38 @@ +sudo: false +language: python +dist: xenial +python: +- '2.7' +- '3.5' +- '3.6' +- '3.7' +addons: + apt_packages: + - libenchant-dev +install: +- pip install tox-travis virtualenv tox python-coveralls coveralls +cache: + directories: + - "$HOME/.cache/pip" +script: +- QUIET=true tox +stages: +- test +- deploy +jobs: + include: + - stage: test + after_success: + - coveralls + - stage: deploy + python: 2.7 + script: skip + install: skip + if: repo = "tranvietanh1991/nameko-django" + deploy: + provider: pypi + user: tranvietanh1991 + password: + secure: ml3I7E9idFy7Yjm6POQOjX4bwobNh23+QhgXfEgOywhF8icbLbB+eRYwVDflC9ekuBiJrWr1aPliXjpWphrGG+z/bawSntv/zERhCjlq3fPXYZnhnGiyZqrfwvq0j67niIBTRUGy51q2Nd9kF5hKivlsLS1XSsodUx2lW4CoeIadLIJIzq9bByVZS3eVGp3e9Nvh598TznWWOu9eKI3lRWEJPNqXQ10K4TmIfQ1y2MyEMw6l8azFKT9raFtKWZO2b92Ie6MXOs+Pf+BuVNZp83FGVxxj6txF43kZseDHyRECfxCj4WgCY78pFfgeBri1lFRX4rKrseMifEx+5YWWbMjn37a456lqxr6UBd15WKr0SHKnZy8eurlPYS8qWWtapnrYCm3W5UAPPsujVFZX25BTTkfcRh9RQxEXzhOXGnpyB1KFBmi/c362c7zOCmzfe3412wZgIBcCchKMrBjoolSxr1rVMUZb4L6I68LLY1Udb0B8Zagyw9GO28PsNm+Dfdgr4DmPn1svXztbWrhsud04C6xFQj9KAdyBN1kgig6kXDUgYmdoSKfUrYi50qzkZ+hwl1EszQy7+Tr7Gs6se0bC+9fYLzNeFiKWHN6YsfVp+4fS1+kTwroZ4m0r9Vo86ZCMO2kfxVI4x4j8t4C9bli1Xzs3GwtXsgwev0U5S14= + on: + branch: master diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f28522f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE README.md MANIFEST.in +include nameko_django/VERSION diff --git a/README.md b/README.md index 21a66bb..b17bf1a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ # nameko-django Django intergration for nameko microservice framework + +## Custom Kombu Serializer for django Object using msgpack and pickle + +This serializer is fully compatible with msgpack so it can be used like this: + +```yaml +serializer: 'django_msgpackpickle' +ACCEPT: ['msgpack', 'django_msgpackpickle'] +SERIALIZERS: + msgpack: + encoder: 'msgpack.dumps' + decoder: 'nameko_django.serializer.loads' + content_type: 'application/x-msgpack' + content_encoding: 'binary' +``` + +In order to migrate an existing microservices stack to use this new serializer first install and setup all project +```yaml +serializer: 'msgpack' +ACCEPT: ['msgpack', 'django_msgpackpickle'] +SERIALIZERS: + msgpack: + encoder: 'msgpack.dumps' + decoder: 'nameko_django.serializer.loads' + content_type: 'application/x-msgpack' + content_encoding: 'binary' +``` +This will accept both of the `msgpack` and `django_msgpackpickle` but only output of result portfolio using `msgpack` +Once all service migrated, then switch to the first configuration diff --git a/nameko_django/VERSION b/nameko_django/VERSION new file mode 100644 index 0000000..7dea76e --- /dev/null +++ b/nameko_django/VERSION @@ -0,0 +1 @@ +1.0.1 diff --git a/nameko_django/__init__.py b/nameko_django/__init__.py new file mode 100644 index 0000000..55708ed --- /dev/null +++ b/nameko_django/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# __init__.py +# +# +# Created by vincenttran on 2019-09-02. +# Copyright (c) 2019 nameko-django. All rights reserved. +# diff --git a/nameko_django/serializer.py b/nameko_django/serializer.py new file mode 100644 index 0000000..fc9169c --- /dev/null +++ b/nameko_django/serializer.py @@ -0,0 +1,144 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# serializer.py +# +# +# Created by vincenttran on 2019-09-02. +# Copyright (c) 2019 bentodatabase. All rights reserved. +# +from __future__ import unicode_literals + +import os +from datetime import datetime +from decimal import Decimal +from io import BytesIO + +from aenum import Enum +from django.db.models import Model, QuerySet +from django.db.models.base import ModelBase +from django.db.models.sql.query import Query +from django.utils import dateparse +from msgpack import packb, unpackb +from six import string_types + +try: + import cPickle as pickle +except ImportError: + import pickle + +DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT = os.getenv("DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT", "%Y-%m-%d %H:%M:%S.%f%z") + + +def serializable(obj): + """ Make an object serializable for JSON, msgpack + + :param obj: Namedtuple instance + :return: + """ + if obj is None: + return + if hasattr(obj, '_asdict') and callable(obj._asdict): + result_obj = dict(obj._asdict()) + elif isinstance(obj, Enum) and hasattr(obj, 'value'): + return obj.value + elif isinstance(obj, Decimal): + return float(obj) + elif isinstance(obj, datetime): + return obj.strftime(DEFAULT_DATETIME_TIMEZONE_STRING_FORMAT) + else: + dump_obj = django_pickle_dumps(obj) + if dump_obj is not None: + return dump_obj + else: + result_obj = obj + + if isinstance(result_obj, dict): + return { + key: serializable(value) + for key, value in result_obj.items() + } + elif isinstance(result_obj, list) or isinstance(result_obj, set) or isinstance(result_obj, tuple): + return [serializable(value) for value in result_obj] + else: + return result_obj + + +def django_is_pickable(s): + if isinstance(s, string_types) and len(s) > 255 and s[:2] == b'\x80\x02' and s[-1] == b'.': + return True + return False + + +def django_pickle_dumps(obj): + if isinstance(obj, Model): + return pickle.dumps(obj, -1) + elif isinstance(obj, QuerySet): + return pickle.dumps((obj.model, obj.query), -1) + else: + return None + + +def django_pickle_loads(obj_string): + objs = pickle.loads(obj_string) + if isinstance(objs, tuple) and len(objs) == 2: + # untouched queryset case + model, query = objs + if isinstance(model, ModelBase) and isinstance(query, Query): + qs = model.objects.all() + qs.query = query + return qs + # normal case + return objs + + +def deserializable(obj): + """ Make an object serializable for JSON, msgpack + + :param obj: Namedtuple instance + :return: + """ + if obj is None: + return + if isinstance(obj, string_types): + if django_is_pickable(obj): + dump_obj = django_pickle_loads(obj) + else: + dump_obj = dateparse.parse_datetime(obj) + if dump_obj is not None: + return dump_obj + else: + result_obj = obj + else: + result_obj = obj + + if isinstance(result_obj, dict): + return { + key: deserializable(value) + for key, value in result_obj.items() + } + elif isinstance(result_obj, list) or isinstance(result_obj, set) or isinstance(result_obj, tuple): + return [deserializable(value) for value in result_obj] + else: + return result_obj + + +def pack(s): + return packb(s, use_bin_type=True) + + +def unpack(s): + return unpackb(s, raw=False) + + +def dumps(o): + return pack(serializable(o)) + + +def loads(s): + if not isinstance(s, string_types): + s = BytesIO(s) + return deserializable(unpack(s)) + + +register_args = (dumps, loads, 'application/x-django-msgpackpickle', 'binary') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..43b645c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +django==1.11.* +nameko==2.11.* +msgpack==0.5.* +tox==3.13.* +flake8 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..452fb24 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,12 @@ +[metadata] +description-file = README.md +license-file = LICENSE +[nosetests] +verbosity=1 +detailed-errors=1 +# with-coverage=1 +# cover-package=coverage +# debug=nose.loader +# pdb=1 +# pdb-failures=1 +stop=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..150d05e --- /dev/null +++ b/setup.py @@ -0,0 +1,61 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# setup.py +# +# +# Created by vincenttran on 2019-09-02. +# Copyright (c) 2019 bentodatabase. All rights reserved. +# +import os + +from os import path + +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), 'rb') as f: + long_description = f.read().decode('utf8') + +with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'nameko_django/VERSION')) as f: + __version__ = f.read() + +from setuptools import setup + +setup( + name='nameko-django', + version=__version__, + description='Django intergration for nameko microservice framework.', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/tranvietanh1991/nameko-django', + author='Vincent Anh Tran', + author_email='tranvietanh1991@gmail.com', + maintainer='Vincent Anh Tran', + maintainer_email='vincent.tran@bentoinvest.cloud', + license='GPLv2', + packages=['nameko_django'], + zip_safe=False, + install_requires=[ + 'nameko>=2.11.0', + 'django>=1.10', + 'msgpack>=0.5.0' + ], + test_suite='nose.collector', + tests_require=['nose'], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + entry_points={ + 'kombu.serializers': [ + 'django_msgpackpickle = nameko_django.serializer:register_args' + ] + } +) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..1037193 --- /dev/null +++ b/tox.ini @@ -0,0 +1,33 @@ +# Tox (http://codespeak.net/~hpk/tox/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. +[flake8] +max-line-length = 121 +exclude = .tox,testsettings*,docs/,bin/,include/,lib/,.git/,*/migrations/*,build/ + + + +[tox] +minversion = 1.8.0 +envlist = + + py{27}-django{111}-nameko{211,212} + flake8 + +toxworkdir = {toxinidir}/.tox + + + +[testenv:package] +deps = twine +commands = + python setup.py sdist + twine check dist/* + +[testenv:flake8] +basepython = python3 +usedevelop = false +deps = flake8 +changedir = {toxinidir} +commands = flake8 nameko_django