Skip to content
Permalink
Browse files

Implemented persistent database connections.

Thanks Anssi Kääriäinen and Karen Tracey for their inputs.
  • Loading branch information...
aaugustin committed Feb 18, 2013
1 parent d009ffe commit 2ee21d9f0d9eaed0494f3b9cd4b5bc9beffffae5
@@ -25,7 +25,7 @@ def check_password(environ, username, password):
return None
return user.check_password(password)
finally:
db.close_connection()
db.close_old_connections()

def groups_for_user(environ, username):
"""
@@ -44,4 +44,4 @@ def groups_for_user(environ, username):
return []
return [force_bytes(group.name) for group in user.groups.all()]
finally:
db.close_connection()
db.close_old_connections()
@@ -42,9 +42,10 @@ def __setattr__(self, name, value):
connection = DefaultConnectionProxy()
backend = load_backend(connection.settings_dict['ENGINE'])

# Register an event that closes the database connection
# when a Django request is finished.
def close_connection(**kwargs):
warnings.warn(
"close_connection is superseded by close_old_connections.",
PendingDeprecationWarning, stacklevel=2)

This comment has been minimized.

Copy link
@ratson

ratson Mar 1, 2013

missing import warnings

This comment has been minimized.

Copy link
@aaugustin

aaugustin Mar 1, 2013

Author Member

Fixed in 203c17c, thanks for noticing.

# Avoid circular imports
from django.db import transaction
for conn in connections:
@@ -53,15 +54,25 @@ def close_connection(**kwargs):
# connection state will be cleaned up.
transaction.abort(conn)
connections[conn].close()
signals.request_finished.connect(close_connection)

# Register an event that resets connection.queries
# when a Django request is started.
# Register an event to reset saved queries when a Django request is started.
def reset_queries(**kwargs):
for conn in connections.all():
conn.queries = []
signals.request_started.connect(reset_queries)

# Register an event to reset transaction state and close connections past
# their lifetime. NB: abort() doesn't do anything outside of a transaction.
def close_old_connections(**kwargs):
for conn in connections.all():
try:
conn.abort()
except DatabaseError:
pass
conn.close_if_unusable_or_obsolete()
signals.request_started.connect(close_old_connections)
signals.request_finished.connect(close_old_connections)

# Register an event that rolls back the connections
# when a Django request has an exception.
def _rollback_on_exception(**kwargs):
@@ -1,4 +1,5 @@
import datetime
import time

from django.db.utils import DatabaseError

@@ -49,6 +50,10 @@ def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS,
self._thread_ident = thread.get_ident()
self.allow_thread_sharing = allow_thread_sharing

# Connection termination related attributes
self.close_at = None
self.errors_occurred = False

def __eq__(self, other):
return self.alias == other.alias

@@ -59,7 +64,7 @@ def __hash__(self):
return hash(self.alias)

def wrap_database_errors(self):
return DatabaseErrorWrapper(self.Database)
return DatabaseErrorWrapper(self)

def get_connection_params(self):
raise NotImplementedError
@@ -76,6 +81,11 @@ def create_cursor(self):
def _cursor(self):
with self.wrap_database_errors():
if self.connection is None:
# Reset parameters defining when to close the connection
max_age = self.settings_dict['CONN_MAX_AGE']
self.close_at = None if max_age is None else time.time() + max_age
self.errors_occurred = False
# Establish the connection
conn_params = self.get_connection_params()
self.connection = self.get_new_connection(conn_params)
self.init_connection_state()
@@ -351,6 +361,26 @@ def close(self):
self.connection = None
self.set_clean()

def close_if_unusable_or_obsolete(self):
if self.connection is not None:
if self.errors_occurred:
if self.is_usable():
self.errors_occurred = False
else:
self.close()
return
if self.close_at is not None and time.time() >= self.close_at:
self.close()
return

def is_usable(self):
"""
Test if the database connection is usable.
This function may assume that self.connection is not None.
"""
raise NotImplementedError

def cursor(self):
self.validate_thread_sharing()
if (self.use_debug_cursor or
@@ -439,6 +439,14 @@ def create_cursor(self):
cursor = self.connection.cursor()
return CursorWrapper(cursor)

def is_usable(self):
try:
self.connection.ping()
except DatabaseError:
return False
else:
return True

def _rollback(self):
try:
BaseDatabaseWrapper._rollback(self)
@@ -598,6 +598,18 @@ def init_connection_state(self):
# stmtcachesize is available only in 4.3.2 and up.
pass

def is_usable(self):
try:
if hasattr(self.connection, 'ping'): # Oracle 10g R2 and higher
self.connection.ping()
else:
# Use a cx_Oracle cursor directly, bypassing Django's utilities.
self.connection.cursor().execute("SELECT 1 FROM DUAL")
except DatabaseError:
return False
else:
return True

# Oracle doesn't support savepoint commits. Ignore them.
def _savepoint_commit(self, sid):
pass
@@ -177,6 +177,15 @@ def create_cursor(self):
cursor.tzinfo_factory = utc_tzinfo_factory if settings.USE_TZ else None
return cursor

def is_usable(self):
try:
# Use a psycopg cursor directly, bypassing Django's utilities.
self.connection.cursor().execute("SELECT 1")
except DatabaseError:
return False
else:
return True

def _enter_transaction_management(self, managed):
"""
Switch the isolation level when needing transaction support, so that
@@ -347,6 +347,9 @@ def init_connection_state(self):
def create_cursor(self):
return self.connection.cursor(factory=SQLiteCursorWrapper)

def is_usable(self):
return True

def check_constraints(self, table_names=None):
"""
Checks each table name in `table_names` for rows with invalid foreign key references. This method is
@@ -56,11 +56,13 @@ class DatabaseErrorWrapper(object):
exceptions using Django's common wrappers.
"""

def __init__(self, database):
def __init__(self, wrapper):
"""
database is a module defining PEP-249 exceptions.
wrapper is a database wrapper.
It must have a Database attribute defining PEP-249 exceptions.
"""
self.database = database
self.wrapper = wrapper

def __enter__(self):
pass
@@ -79,7 +81,7 @@ def __exit__(self, exc_type, exc_value, traceback):
InterfaceError,
Error,
):
db_exc_type = getattr(self.database, dj_exc_type.__name__)
db_exc_type = getattr(self.wrapper.Database, dj_exc_type.__name__)
if issubclass(exc_type, db_exc_type):
# Under Python 2.6, exc_value can still be a string.
try:
@@ -89,6 +91,10 @@ def __exit__(self, exc_type, exc_value, traceback):
dj_exc_value = dj_exc_type(*args)
if six.PY3:
dj_exc_value.__cause__ = exc_value
# Only set the 'errors_occurred' flag for errors that may make
# the connection unusable.
if dj_exc_type not in (DataError, IntegrityError):
self.wrapper.errors_occurred = True
six.reraise(dj_exc_type, dj_exc_value, traceback)

def __call__(self, func):
@@ -155,6 +161,7 @@ def ensure_defaults(self, alias):
conn.setdefault('ENGINE', 'django.db.backends.dummy')
if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']:
conn['ENGINE'] = 'django.db.backends.dummy'
conn.setdefault('CONN_MAX_AGE', 600)
conn.setdefault('OPTIONS', {})
conn.setdefault('TIME_ZONE', 'UTC' if settings.USE_TZ else settings.TIME_ZONE)
for setting in ['NAME', 'USER', 'PASSWORD', 'HOST', 'PORT']:
@@ -18,7 +18,7 @@
from django.core.handlers.wsgi import WSGIRequest
from django.core.signals import (request_started, request_finished,
got_request_exception)
from django.db import close_connection
from django.db import close_old_connections
from django.http import SimpleCookie, HttpRequest, QueryDict
from django.template import TemplateDoesNotExist
from django.test import signals
@@ -78,9 +78,9 @@ def closing_iterator_wrapper(iterable, close):
for item in iterable:
yield item
finally:
request_finished.disconnect(close_connection)
request_finished.disconnect(close_old_connections)
close() # will fire request_finished
request_finished.connect(close_connection)
request_finished.connect(close_old_connections)


class ClientHandler(BaseHandler):
@@ -101,7 +101,9 @@ def __call__(self, environ):
if self._request_middleware is None:
self.load_middleware()

request_started.disconnect(close_old_connections)
request_started.send(sender=self.__class__)
request_started.connect(close_old_connections)
request = WSGIRequest(environ)
# sneaky little hack so that we can easily get round
# CsrfViewMiddleware. This makes life easier, and is probably
@@ -115,9 +117,9 @@ def __call__(self, environ):
response.streaming_content = closing_iterator_wrapper(
response.streaming_content, response.close)
else:
request_finished.disconnect(close_connection)
request_finished.disconnect(close_old_connections)
response.close() # will fire request_finished
request_finished.connect(close_connection)
request_finished.connect(close_old_connections)

return response

@@ -339,6 +339,8 @@ these changes.

* ``Model._meta.module_name`` was renamed to ``model_name``.

* The private API ``django.db.close_connection`` will be removed.

2.0
---

@@ -11,6 +11,68 @@ This file describes some of the features that might be relevant to Django
usage. Of course, it is not intended as a replacement for server-specific
documentation or reference manuals.

General notes
=============

.. _persistent-database-connections:

Persistent connections
----------------------

.. versionadded:: 1.6

Persistent connections avoid the overhead of re-establishing a connection to
the database in each request. By default, connections are kept open for up 10
minutes — if not specified, :setting:`CONN_MAX_AGE` defaults to 600 seconds.

Django 1.5 and earlier didn't have persistent connections. To restore the
legacy behavior of closing the connection at the end of every request, set
:setting:`CONN_MAX_AGE` to ``0``.

For unlimited persistent connections, set :setting:`CONN_MAX_AGE` to ``None``.

Connection management
~~~~~~~~~~~~~~~~~~~~~

Django opens a connection to the database when it first makes a database
query. It keeps this connection open and reuses it in subsequent requests.
Django closes the connection once it exceeds the maximum age defined by
:setting:`CONN_MAX_AGE` or when it isn't usable any longer.

In detail, Django automatically opens a connection to the database whenever it
needs one and doesn't have one already — either because this is the first
connection, or because the previous connection was closed.

At the beginning of each request, Django closes the connection if it has
reached its maximum age. If your database terminates idle connections after
some time, you should set :setting:`CONN_MAX_AGE` to a lower value, so that
Django doesn't attempt to use a connection that has been terminated by the
database server. (This problem may only affect very low traffic sites.)

At the end of each request, Django closes the connection if it has reached its
maximum age or if it is in an unrecoverable error state. If any database
errors have occurred while processing the requests, Django checks whether the
connection still works, and closes it if it doesn't. Thus, database errors
affect at most one request; if the connection becomes unusable, the next
request gets a fresh connection.

Caveats
~~~~~~~

Since each thread maintains its own connection, your database must support at
least as many simultaneous connections as you have worker threads.

Sometimes a database won't be accessed by the majority of your views, for
example because it's the database of an external system, or thanks to caching.
In such cases, you should set :setting:`CONN_MAX_AGE` to a lower value, or
even ``0``, because it doesn't make sense to maintain a connection that's
unlikely to be reused. This will help keep the number of simultaneous
connections to this database small.


The development server creates a new thread for each request it handles,
negating the effect of persistent connections.

.. _postgresql-notes:

PostgreSQL notes
@@ -464,6 +464,19 @@ The name of the database to use. For SQLite, it's the full path to the database
file. When specifying the path, always use forward slashes, even on Windows
(e.g. ``C:/homes/user/mysite/sqlite3.db``).

.. setting:: CONN_MAX_AGE

CONN_MAX_AGE
~~~~~~~~~~~~

.. versionadded:: 1.6

Default: ``600``

The lifetime of a database connection, in seconds. Use ``0`` to close database
connections at the end of each request — Django's historical behavior — and
``None`` for unlimited persistent connections.

.. setting:: OPTIONS

OPTIONS

0 comments on commit 2ee21d9

Please sign in to comment.
You can’t perform that action at this time.