Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Implemented persistent database connections.

Thanks Anssi Kääriäinen and Karen Tracey for their inputs.
  • Loading branch information...
commit 2ee21d9f0d9eaed0494f3b9cd4b5bc9beffffae5 1 parent d009ffe
@aaugustin aaugustin authored
View
4 django/contrib/auth/handlers/modwsgi.py
@@ -25,7 +25,7 @@ def check_password(environ, username, password):
return None
return user.check_password(password)
finally:
- db.close_connection()
+ db.close_old_connections()
def groups_for_user(environ, username):
"""
@@ -44,4 +44,4 @@ def groups_for_user(environ, username):
return []
return [force_bytes(group.name) for group in user.groups.all()]
finally:
- db.close_connection()
+ db.close_old_connections()
View
21 django/db/__init__.py
@@ -42,9 +42,10 @@ def __setattr__(self, name, value):
connection = DefaultConnectionProxy()
backend = load_backend(connection.settings_dict['ENGINE'])
-# Register an event that closes the database connection
-# when a Django request is finished.
def close_connection(**kwargs):
+ warnings.warn(
+ "close_connection is superseded by close_old_connections.",
+ PendingDeprecationWarning, stacklevel=2)
@r4ts0n
r4ts0n added a note

missing import warnings

@aaugustin Owner

Fixed in 203c17c, thanks for noticing.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
# Avoid circular imports
from django.db import transaction
for conn in connections:
@@ -53,15 +54,25 @@ def close_connection(**kwargs):
# connection state will be cleaned up.
transaction.abort(conn)
connections[conn].close()
-signals.request_finished.connect(close_connection)
-# Register an event that resets connection.queries
-# when a Django request is started.
+# Register an event to reset saved queries when a Django request is started.
def reset_queries(**kwargs):
for conn in connections.all():
conn.queries = []
signals.request_started.connect(reset_queries)
+# Register an event to reset transaction state and close connections past
+# their lifetime. NB: abort() doesn't do anything outside of a transaction.
+def close_old_connections(**kwargs):
+ for conn in connections.all():
+ try:
+ conn.abort()
+ except DatabaseError:
+ pass
+ conn.close_if_unusable_or_obsolete()
+signals.request_started.connect(close_old_connections)
+signals.request_finished.connect(close_old_connections)
+
# Register an event that rolls back the connections
# when a Django request has an exception.
def _rollback_on_exception(**kwargs):
View
32 django/db/backends/__init__.py
@@ -1,4 +1,5 @@
import datetime
+import time
from django.db.utils import DatabaseError
@@ -49,6 +50,10 @@ def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS,
self._thread_ident = thread.get_ident()
self.allow_thread_sharing = allow_thread_sharing
+ # Connection termination related attributes
+ self.close_at = None
+ self.errors_occurred = False
+
def __eq__(self, other):
return self.alias == other.alias
@@ -59,7 +64,7 @@ def __hash__(self):
return hash(self.alias)
def wrap_database_errors(self):
- return DatabaseErrorWrapper(self.Database)
+ return DatabaseErrorWrapper(self)
def get_connection_params(self):
raise NotImplementedError
@@ -76,6 +81,11 @@ def create_cursor(self):
def _cursor(self):
with self.wrap_database_errors():
if self.connection is None:
+ # Reset parameters defining when to close the connection
+ max_age = self.settings_dict['CONN_MAX_AGE']
+ self.close_at = None if max_age is None else time.time() + max_age
+ self.errors_occurred = False
+ # Establish the connection
conn_params = self.get_connection_params()
self.connection = self.get_new_connection(conn_params)
self.init_connection_state()
@@ -351,6 +361,26 @@ def close(self):
self.connection = None
self.set_clean()
+ def close_if_unusable_or_obsolete(self):
+ if self.connection is not None:
+ if self.errors_occurred:
+ if self.is_usable():
+ self.errors_occurred = False
+ else:
+ self.close()
+ return
+ if self.close_at is not None and time.time() >= self.close_at:
+ self.close()
+ return
+
+ def is_usable(self):
+ """
+ Test if the database connection is usable.
+
+ This function may assume that self.connection is not None.
+ """
+ raise NotImplementedError
+
def cursor(self):
self.validate_thread_sharing()
if (self.use_debug_cursor or
View
8 django/db/backends/mysql/base.py
@@ -439,6 +439,14 @@ def create_cursor(self):
cursor = self.connection.cursor()
return CursorWrapper(cursor)
+ def is_usable(self):
+ try:
+ self.connection.ping()
+ except DatabaseError:
+ return False
+ else:
+ return True
+
def _rollback(self):
try:
BaseDatabaseWrapper._rollback(self)
View
12 django/db/backends/oracle/base.py
@@ -598,6 +598,18 @@ def init_connection_state(self):
# stmtcachesize is available only in 4.3.2 and up.
pass
+ def is_usable(self):
+ try:
+ if hasattr(self.connection, 'ping'): # Oracle 10g R2 and higher
+ self.connection.ping()
+ else:
+ # Use a cx_Oracle cursor directly, bypassing Django's utilities.
+ self.connection.cursor().execute("SELECT 1 FROM DUAL")
+ except DatabaseError:
+ return False
+ else:
+ return True
+
# Oracle doesn't support savepoint commits. Ignore them.
def _savepoint_commit(self, sid):
pass
View
9 django/db/backends/postgresql_psycopg2/base.py
@@ -177,6 +177,15 @@ def create_cursor(self):
cursor.tzinfo_factory = utc_tzinfo_factory if settings.USE_TZ else None
return cursor
+ def is_usable(self):
+ try:
+ # Use a psycopg cursor directly, bypassing Django's utilities.
+ self.connection.cursor().execute("SELECT 1")
+ except DatabaseError:
+ return False
+ else:
+ return True
+
def _enter_transaction_management(self, managed):
"""
Switch the isolation level when needing transaction support, so that
View
3  django/db/backends/sqlite3/base.py
@@ -347,6 +347,9 @@ def init_connection_state(self):
def create_cursor(self):
return self.connection.cursor(factory=SQLiteCursorWrapper)
+ def is_usable(self):
+ return True
+
def check_constraints(self, table_names=None):
"""
Checks each table name in `table_names` for rows with invalid foreign key references. This method is
View
15 django/db/utils.py
@@ -56,11 +56,13 @@ class DatabaseErrorWrapper(object):
exceptions using Django's common wrappers.
"""
- def __init__(self, database):
+ def __init__(self, wrapper):
"""
- database is a module defining PEP-249 exceptions.
+ wrapper is a database wrapper.
+
+ It must have a Database attribute defining PEP-249 exceptions.
"""
- self.database = database
+ self.wrapper = wrapper
def __enter__(self):
pass
@@ -79,7 +81,7 @@ def __exit__(self, exc_type, exc_value, traceback):
InterfaceError,
Error,
):
- db_exc_type = getattr(self.database, dj_exc_type.__name__)
+ db_exc_type = getattr(self.wrapper.Database, dj_exc_type.__name__)
if issubclass(exc_type, db_exc_type):
# Under Python 2.6, exc_value can still be a string.
try:
@@ -89,6 +91,10 @@ def __exit__(self, exc_type, exc_value, traceback):
dj_exc_value = dj_exc_type(*args)
if six.PY3:
dj_exc_value.__cause__ = exc_value
+ # Only set the 'errors_occurred' flag for errors that may make
+ # the connection unusable.
+ if dj_exc_type not in (DataError, IntegrityError):
+ self.wrapper.errors_occurred = True
six.reraise(dj_exc_type, dj_exc_value, traceback)
def __call__(self, func):
@@ -155,6 +161,7 @@ def ensure_defaults(self, alias):
conn.setdefault('ENGINE', 'django.db.backends.dummy')
if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']:
conn['ENGINE'] = 'django.db.backends.dummy'
+ conn.setdefault('CONN_MAX_AGE', 600)
conn.setdefault('OPTIONS', {})
conn.setdefault('TIME_ZONE', 'UTC' if settings.USE_TZ else settings.TIME_ZONE)
for setting in ['NAME', 'USER', 'PASSWORD', 'HOST', 'PORT']:
View
12 django/test/client.py
@@ -18,7 +18,7 @@
from django.core.handlers.wsgi import WSGIRequest
from django.core.signals import (request_started, request_finished,
got_request_exception)
-from django.db import close_connection
+from django.db import close_old_connections
from django.http import SimpleCookie, HttpRequest, QueryDict
from django.template import TemplateDoesNotExist
from django.test import signals
@@ -78,9 +78,9 @@ def closing_iterator_wrapper(iterable, close):
for item in iterable:
yield item
finally:
- request_finished.disconnect(close_connection)
+ request_finished.disconnect(close_old_connections)
close() # will fire request_finished
- request_finished.connect(close_connection)
+ request_finished.connect(close_old_connections)
class ClientHandler(BaseHandler):
@@ -101,7 +101,9 @@ def __call__(self, environ):
if self._request_middleware is None:
self.load_middleware()
+ request_started.disconnect(close_old_connections)
request_started.send(sender=self.__class__)
+ request_started.connect(close_old_connections)
request = WSGIRequest(environ)
# sneaky little hack so that we can easily get round
# CsrfViewMiddleware. This makes life easier, and is probably
@@ -115,9 +117,9 @@ def __call__(self, environ):
response.streaming_content = closing_iterator_wrapper(
response.streaming_content, response.close)
else:
- request_finished.disconnect(close_connection)
+ request_finished.disconnect(close_old_connections)
response.close() # will fire request_finished
- request_finished.connect(close_connection)
+ request_finished.connect(close_old_connections)
return response
View
2  docs/internals/deprecation.txt
@@ -339,6 +339,8 @@ these changes.
* ``Model._meta.module_name`` was renamed to ``model_name``.
+* The private API ``django.db.close_connection`` will be removed.
+
2.0
---
View
62 docs/ref/databases.txt
@@ -11,6 +11,68 @@ This file describes some of the features that might be relevant to Django
usage. Of course, it is not intended as a replacement for server-specific
documentation or reference manuals.
+General notes
+=============
+
+.. _persistent-database-connections:
+
+Persistent connections
+----------------------
+
+.. versionadded:: 1.6
+
+Persistent connections avoid the overhead of re-establishing a connection to
+the database in each request. By default, connections are kept open for up 10
+minutes — if not specified, :setting:`CONN_MAX_AGE` defaults to 600 seconds.
+
+Django 1.5 and earlier didn't have persistent connections. To restore the
+legacy behavior of closing the connection at the end of every request, set
+:setting:`CONN_MAX_AGE` to ``0``.
+
+For unlimited persistent connections, set :setting:`CONN_MAX_AGE` to ``None``.
+
+Connection management
+~~~~~~~~~~~~~~~~~~~~~
+
+Django opens a connection to the database when it first makes a database
+query. It keeps this connection open and reuses it in subsequent requests.
+Django closes the connection once it exceeds the maximum age defined by
+:setting:`CONN_MAX_AGE` or when it isn't usable any longer.
+
+In detail, Django automatically opens a connection to the database whenever it
+needs one and doesn't have one already — either because this is the first
+connection, or because the previous connection was closed.
+
+At the beginning of each request, Django closes the connection if it has
+reached its maximum age. If your database terminates idle connections after
+some time, you should set :setting:`CONN_MAX_AGE` to a lower value, so that
+Django doesn't attempt to use a connection that has been terminated by the
+database server. (This problem may only affect very low traffic sites.)
+
+At the end of each request, Django closes the connection if it has reached its
+maximum age or if it is in an unrecoverable error state. If any database
+errors have occurred while processing the requests, Django checks whether the
+connection still works, and closes it if it doesn't. Thus, database errors
+affect at most one request; if the connection becomes unusable, the next
+request gets a fresh connection.
+
+Caveats
+~~~~~~~
+
+Since each thread maintains its own connection, your database must support at
+least as many simultaneous connections as you have worker threads.
+
+Sometimes a database won't be accessed by the majority of your views, for
+example because it's the database of an external system, or thanks to caching.
+In such cases, you should set :setting:`CONN_MAX_AGE` to a lower value, or
+even ``0``, because it doesn't make sense to maintain a connection that's
+unlikely to be reused. This will help keep the number of simultaneous
+connections to this database small.
+
+
+The development server creates a new thread for each request it handles,
+negating the effect of persistent connections.
+
.. _postgresql-notes:
PostgreSQL notes
View
13 docs/ref/settings.txt
@@ -464,6 +464,19 @@ The name of the database to use. For SQLite, it's the full path to the database
file. When specifying the path, always use forward slashes, even on Windows
(e.g. ``C:/homes/user/mysite/sqlite3.db``).
+.. setting:: CONN_MAX_AGE
+
+CONN_MAX_AGE
+~~~~~~~~~~~~
+
+.. versionadded:: 1.6
+
+Default: ``600``
+
+The lifetime of a database connection, in seconds. Use ``0`` to close database
+connections at the end of each request — Django's historical behavior — and
+``None`` for unlimited persistent connections.
+
.. setting:: OPTIONS
OPTIONS
View
21 docs/releases/1.6.txt
@@ -30,6 +30,19 @@ prevention <clickjacking-prevention>` are turned on.
If the default templates don't suit your tastes, you can use :ref:`custom
project and app templates <custom-app-and-project-templates>`.
+Persistent database connections
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Django now supports reusing the same database connection for several requests.
+This avoids the overhead of re-establishing a connection at the beginning of
+each request.
+
+By default, database connections will kept open for 10 minutes. This behavior
+is controlled by the :setting:`CONN_MAX_AGE` setting. To restore the previous
+behavior of closing the connection at the end of each request, set
+:setting:`CONN_MAX_AGE` to ``0``. See :ref:`persistent-database-connections`
+for details.
+
Time zone aware aggregation
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -136,6 +149,14 @@ Backwards incompatible changes in 1.6
* Model fields named ``hour``, ``minute`` or ``second`` may clash with the new
lookups. Append an explicit :lookup:`exact` lookup if this is an issue.
+* When Django establishes a connection to the database, it sets up appropriate
+ parameters, depending on the backend being used. Since `persistent database
+ connections <persistent-database-connections>`_ are enabled by default in
+ Django 1.6, this setup isn't repeated at every request any more. If you
+ modifiy parameters such as the connection's isolation level or time zone,
+ you should either restore Django's defaults at the end of each request, or
+ force an appropriate value at the beginning of each request.
+
* If your CSS/Javascript code used to access HTML input widgets by type, you
should review it as ``type='text'`` widgets might be now output as
``type='email'``, ``type='url'`` or ``type='number'`` depending on their
View
17 tests/handlers/tests.py
@@ -1,5 +1,6 @@
from django.core.handlers.wsgi import WSGIHandler
-from django.core import signals
+from django.core.signals import request_started, request_finished
+from django.db import close_old_connections
from django.test import RequestFactory, TestCase
from django.test.utils import override_settings
from django.utils import six
@@ -7,6 +8,12 @@
class HandlerTests(TestCase):
+ def setUp(self):
+ request_started.disconnect(close_old_connections)
+
+ def tearDown(self):
+ request_started.connect(close_old_connections)
+
# Mangle settings so the handler will fail
@override_settings(MIDDLEWARE_CLASSES=42)
def test_lock_safety(self):
@@ -35,12 +42,12 @@ class SignalsTests(TestCase):
def setUp(self):
self.signals = []
- signals.request_started.connect(self.register_started)
- signals.request_finished.connect(self.register_finished)
+ request_started.connect(self.register_started)
+ request_finished.connect(self.register_finished)
def tearDown(self):
- signals.request_started.disconnect(self.register_started)
- signals.request_finished.disconnect(self.register_finished)
+ request_started.disconnect(self.register_started)
+ request_finished.disconnect(self.register_finished)
def register_started(self, **kwargs):
self.signals.append('started')
View
6 tests/httpwrappers/tests.py
@@ -8,7 +8,7 @@
from django.core.exceptions import SuspiciousOperation
from django.core.signals import request_finished
-from django.db import close_connection
+from django.db import close_old_connections
from django.http import (QueryDict, HttpResponse, HttpResponseRedirect,
HttpResponsePermanentRedirect, HttpResponseNotAllowed,
HttpResponseNotModified, StreamingHttpResponse,
@@ -490,10 +490,10 @@ class FileCloseTests(TestCase):
def setUp(self):
# Disable the request_finished signal during this test
# to avoid interfering with the database connection.
- request_finished.disconnect(close_connection)
+ request_finished.disconnect(close_old_connections)
def tearDown(self):
- request_finished.connect(close_connection)
+ request_finished.connect(close_old_connections)
def test_response(self):
filename = os.path.join(os.path.dirname(upath(__file__)), 'abc.txt')
View
8 tests/wsgi/tests.py
@@ -2,7 +2,9 @@
from django.core.exceptions import ImproperlyConfigured
from django.core.servers.basehttp import get_internal_wsgi_application
+from django.core.signals import request_started
from django.core.wsgi import get_wsgi_application
+from django.db import close_old_connections
from django.test import TestCase
from django.test.client import RequestFactory
from django.test.utils import override_settings
@@ -12,6 +14,12 @@
class WSGITest(TestCase):
urls = "wsgi.urls"
+ def setUp(self):
+ request_started.disconnect(close_old_connections)
+
+ def tearDown(self):
+ request_started.connect(close_old_connections)
+
def test_get_wsgi_application(self):
"""
Verify that ``get_wsgi_application`` returns a functioning WSGI
Please sign in to comment.
Something went wrong with that request. Please try again.