initial commit

.gitignore

@@ -0,0 +1,6 @@
+*.iml
+*.pyc
+*.pyd
+build/
+dist/
+*.egg-info/

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2018, Jonas Maurus
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.rst

@@ -0,0 +1,67 @@
+Django Database Connection Autoreconnect
+========================================
+
+This library monkeypatches ``django.db.backends.base.BaseDatabaseWrapper`` so
+that when a database operation fails because the underlying TCP connection was
+already closed, it first tried to reconnect, instead of immediately raising
+an ``OperationException``.
+
+
+Why is this useful?
+-------------------
+I use `HAProxy`__ as a load-balancer in front of my PostgreSQL databases all
+the time, sometimes in addition to `pgbouncer`. Even though you can mostly 
+prevent surprises by enabling TCP keep-alive packets through ``tcpka``__, 
+``clitcpka``__ and ``srvtcpka``__, I still encounter situations where the
+underlying TCP connection has been closed through the load-balancer. Most often
+this results in
+
+    django.db.utils.OperationalError: server closed the connection unexpectedly
+    This probably means the server terminated abnormally before or while
+    processing the request.
+
+This library patches Django such that it try to reconnect once before failing.
+
+Another application of this is when using `Hashicorp Vault`__, where
+credentials for a database connection can expire at any time and then need to
+be refreshed from Vault.
+
+
+How to install?
+---------------
+Just pull the library in using ``pip install django-dbconn-retry``. Then add
+``django_dbconn_retry`` to ``INSTALLED_APPS`` in your ``settings.py``.
+
+
+Provided hooks
+--------------
+The library provides an interface for other code to plug into the process to,
+for example, allow ``12factor-vault``__ to refresh the database credentials
+before the code tries to reestablish the database connection.
+
+===========================  ==================================================
+Hook                         Description
+===========================  ==================================================
+``add_pre_reconnect_hook``   Installs a hook of the type 
+                             ``Callable[[BaseDatabaseWrapper], None]`` that
+                             will be called before the library tries to
+                             reestablish a connection. 12factor-vault uses this
+                             to refresh the database credentials from Vault.
+``add_post_reconnect_hook``  Installs a hook of the type
+                             ``Callable[[BaseDatabaseWrapper], None]`` that
+                             will be called after the library tried to
+                             reestablish the connection. Success or failure has
+                             not been tested at this point. So the connection
+                             may be in any state.
+===========================  ==================================================
+
+
+
+.. _HAProxy: http://www.haproxy.org/
+.. _tcpka:
+   https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#option%20tcpka
+.. _clitcpka: 
+   https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-option%20clitcpka
+.. _srvtcpka:
+   https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#option%20srvtcpka
+.. _Hashicorp Vault: https://vaultproject.io/

django_dbconn_retry/__init__.py

@@ -0,0 +1,96 @@
+# -* encoding: utf-8 *-
+import logging
+
+from django.apps.config import AppConfig
+from django.db import utils as django_db_utils
+from django.db.backends.base import base as django_db_base
+
+from typing import Union, Tuple, Callable, List
+
+_log = logging.getLogger(__name__)
+default_app_config = 'django_dbconn_retry.DjangoIntegration'
+
+pre_reconnect_hooks = []  # type: List[Callable[[django_db_base.BaseDatabaseWrapper], None]]
+post_reconnect_hooks = []  # type: List[Callable[[django_db_base.BaseDatabaseWrapper], None]]
+
+
+_operror_types = ()  # type: Union[Tuple[type], Tuple]
+_operror_types += (django_db_utils.OperationalError,)
+try:
+    import psycopg2
+except ImportError:
+    pass
+else:
+    _operror_types += (psycopg2.OperationalError,)
+
+try:
+    import sqlite3
+except ImportError:
+    pass
+else:
+    _operror_types += (sqlite3.OperationalError,)
+
+try:
+    import MySQLdb
+except ImportError:
+    pass
+else:
+    _operror_types += (MySQLdb.OperationalError,)
+
+
+def add_pre_reconnect_hook(hook: Callable[[django_db_base.BaseDatabaseWrapper], None]) -> None:
+    global pre_reconnect_hooks
+    pre_reconnect_hooks.append(hook)
+
+
+def add_post_reconnect_hook(hook: Callable[[django_db_base.BaseDatabaseWrapper], None]) -> None:
+    global post_reconnect_hooks
+    post_reconnect_hooks.append(hook)
+
+
+def monkeypatch_django() -> None:
+    global pre_reconnect_hooks, post_reconnect_hooks
+
+    def ensure_connection_with_retries(self: django_db_base.BaseDatabaseWrapper) -> None:
+        if self.connection is not None and self.connection.closed:
+            _log.debug("failed connection detected")
+            self.connection = None
+
+        if self.connection is None:
+            with self.wrap_database_errors:
+                try:
+                    self.connect()
+                except Exception as e:
+                    if isinstance(e, _operror_types):
+                        if hasattr(self, "_connection_retries") and self._connection_retries >= 1:
+                            _log.error("Reconnecting to the database didn't help %s", str(e))
+                            raise
+                        else:
+                            _log.info("Database connection failed. Refreshing...")
+                            self._connection_retries = 1
+
+                            for hook in pre_reconnect_hooks:
+                                hook(self)
+
+                            self.ensure_connection()
+
+                            for hook in post_reconnect_hooks:
+                                hook(self)
+                    else:
+                        _log.debug("Database connection failed, but not due to a known error for vault12factor %s",
+                                   str(e))
+                        raise
+                else:
+                    self._connection_retries = 0
+
+    _log.debug("12factor-vault: monkeypatching BaseDatabaseWrapper")
+    django_db_base.BaseDatabaseWrapper.ensure_connection = ensure_connection_with_retries
+
+
+class DjangoIntegration(AppConfig):
+    name = "django_dbconn_retry"
+
+    def ready(self) -> None:
+        monkeypatch_django()
+
+

setup.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# -* encoding: utf-8 *-
+import os
+from setuptools import setup
+
+HERE = os.path.dirname(__file__)
+
+try:
+    long_description = open(os.path.join(HERE, 'README.rst')).read()
+except IOError:
+    long_description = None
+
+
+setup(
+    name="django-dbconn-retry",
+    version="0.1.0",
+    packages=["django_dbconn_retry"],
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Developers",
+        "Intended Audience :: System Administrators",
+        "Programming Language :: Python :: 3 :: Only",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: POSIX",
+    ],
+    url="https://github.com/jdelic/django-dbconn-retry/",
+    author="Jonas Maurus (@jdelic)",
+    author_email="jonas-12factor-vault@gopythongo.com",
+    maintainer="GoPythonGo.com",
+    maintainer_email="info@gopythongo.com",
+    description="Patch Django to retry a database connection first before failing.",
+    long_description=long_description,
+
+    install_requires=[
+    ],
+)