Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Implemented persistent database connections.

Thanks Anssi Kääriäinen and Karen Tracey for their inputs.
  • Loading branch information...
commit 2ee21d9f0d9eaed0494f3b9cd4b5bc9beffffae5 1 parent d009ffe
@aaugustin aaugustin authored
View
4 django/contrib/auth/handlers/modwsgi.py
@@ -25,7 +25,7 @@ def check_password(environ, username, password):
return None
return user.check_password(password)
finally:
- db.close_connection()
+ db.close_old_connections()
def groups_for_user(environ, username):
"""
@@ -44,4 +44,4 @@ def groups_for_user(environ, username):
return []
return [force_bytes(group.name) for group in user.groups.all()]
finally:
- db.close_connection()
+ db.close_old_connections()
View
21 django/db/__init__.py
@@ -42,9 +42,10 @@ def __setattr__(self, name, value):
connection = DefaultConnectionProxy()
backend = load_backend(connection.settings_dict['ENGINE'])
-# Register an event that closes the database connection
-# when a Django request is finished.
def close_connection(**kwargs):
+ warnings.warn(
+ "close_connection is superseded by close_old_connections.",
+ PendingDeprecationWarning, stacklevel=2)
@r4ts0n
r4ts0n added a note

missing import warnings

@aaugustin Owner

Fixed in 203c17c, thanks for noticing.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
# Avoid circular imports
from django.db import transaction
for conn in connections:
@@ -53,15 +54,25 @@ def close_connection(**kwargs):
# connection state will be cleaned up.
transaction.abort(conn)
connections[conn].close()
-signals.request_finished.connect(close_connection)
-# Register an event that resets connection.queries
-# when a Django request is started.
+# Register an event to reset saved queries when a Django request is started.
def reset_queries(**kwargs):
for conn in connections.all():
conn.queries = []
signals.request_started.connect(reset_queries)
+# Register an event to reset transaction state and close connections past
+# their lifetime. NB: abort() doesn't do anything outside of a transaction.
+def close_old_connections(**kwargs):
+ for conn in connections.all():
+ try:
+ conn.abort()
+ except DatabaseError:
+ pass
+ conn.close_if_unusable_or_obsolete()
+signals.request_started.connect(close_old_connections)
+signals.request_finished.connect(close_old_connections)
+
# Register an event that rolls back the connections
# when a Django request has an exception.
def _rollback_on_exception(**kwargs):
View
32 django/db/backends/__init__.py
@@ -1,4 +1,5 @@
import datetime
+import time
from django.db.utils import DatabaseError
@@ -49,6 +50,10 @@ def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS,
self._thread_ident = thread.get_ident()
self.allow_thread_sharing = allow_thread_sharing
+ # Connection termination related attributes
+ self.close_at = None
+ self.errors_occurred = False
+
def __eq__(self, other):
return self.alias == other.alias
@@ -59,7 +64,7 @@ def __hash__(self):
return hash(self.alias)
def wrap_database_errors(self):
- return DatabaseErrorWrapper(self.Database)
+ return DatabaseErrorWrapper(self)
def get_connection_params(self):
raise NotImplementedError
@@ -76,6 +81,11 @@ def create_cursor(self):
def _cursor(self):
with self.wrap_database_errors():
if self.connection is None:
+ # Reset parameters defining when to close the connection
+ max_age = self.settings_dict['CONN_MAX_AGE']
+ self.close_at = None if max_age is None else time.time() + max_age
+ self.errors_occurred = False
+ # Establish the connection
conn_params = self.get_connection_params()
self.connection = self.get_new_connection(conn_params)
self.init_connection_state()
@@ -351,6 +361,26 @@ def close(self):
self.connection = None
self.set_clean()
+ def close_if_unusable_or_obsolete(self):
+ if self.connection is not None:
+ if self.errors_occurred:
+ if self.is_usable():
+ self.errors_occurred = False
+ else:
+ self.close()
+ return
+ if self.close_at is not None and time.time() >= self.close_at:
+ self.close()
+ return
+
+ def is_usable(self):
+ """
+ Test if the database connection is usable.
+
+ This function may assume that self.connection is not None.
+ """
+ raise NotImplementedError
+
def cursor(self):
self.validate_thread_sharing()
if (self.use_debug_cursor or
View
8 django/db/backends/mysql/base.py
@@ -439,6 +439,14 @@ def create_cursor(self):
cursor = self.connection.cursor()
return CursorWrapper(cursor)
+ def is_usable(self):
+ try:
+ self.connection.ping()
+ except DatabaseError:
+ return False
+ else:
+ return True
+
def _rollback(self):
try:
BaseDatabaseWrapper._rollback(self)
View
12 django/db/backends/oracle/base.py
@@ -598,6 +598,18 @@ def init_connection_state(self):
# stmtcachesize is available only in 4.3.2 and up.
pass
+ def is_usable(self):
+ try:
+ if hasattr(self.connection, 'ping'): # Oracle 10g R2 and higher
+ self.connection.ping()
+ else:
+ # Use a cx_Oracle cursor directly, bypassing Django's utilities.
+ self.connection.cursor().execute("SELECT 1 FROM DUAL")
+ except DatabaseError:
+ return False
+ else:
+ return True
+
# Oracle doesn't support savepoint commits. Ignore them.
def _savepoint_commit(self, sid):
pass
View
9 django/db/backends/postgresql_psycopg2/base.py
@@ -177,6 +177,15 @@ def create_cursor(self):
cursor.tzinfo_factory = utc_tzinfo_factory if settings.USE_TZ else None
return cursor
+ def is_usable(self):
+ try:
+ # Use a psycopg cursor directly, bypassing Django's utilities.
+ self.connection.cursor().execute("SELECT 1")
+ except DatabaseError:
+ return False
+ else:
+ return True
+
def _enter_transaction_management(self, managed):
"""
Switch the isolation level when needing transaction support, so that
View
3  django/db/backends/sqlite3/base.py
@@ -347,6 +347,9 @@ def init_connection_state(self):
def create_cursor(self):
return self.connection.cursor(factory=SQLiteCursorWrapper)
+ def is_usable(self):
+ return True
+
def check_constraints(self, table_names=None):
"""
Checks each table name in `table_names` for rows with invalid foreign key references. This method is
View
15 django/db/utils.py
@@ -56,11 +56,13 @@ class DatabaseErrorWrapper(object):
exceptions using Django's common wrappers.
"""
- def __init__(self, database):
+ def __init__(self, wrapper):
"""
- database is a module defining PEP-249 exceptions.
+ wrapper is a database wrapper.
+
+ It must have a Database attribute defining PEP-249 exceptions.
"""
- self.database = database
+ self.wrapper = wrapper
def __enter__(self):
pass
@@ -79,7 +81,7 @@ def __exit__(self, exc_type, exc_value, traceback):
InterfaceError,
Error,
):
- db_exc_type = getattr(self.database, dj_exc_type.__name__)
+ db_exc_type = getattr(self.wrapper.Database, dj_exc_type.__name__)
if issubclass(exc_type, db_exc_type):
# Under Python 2.6, exc_value can still be a string.
try:
@@ -89,6 +91,10 @@ def __exit__(self, exc_type, exc_value, traceback):
dj_exc_value = dj_exc_type(*args)
if six.PY3:
dj_exc_value.__cause__ = exc_value
+ # Only set the 'errors_occurred' flag for errors that may make
+ # the connection unusable.
+ if dj_exc_type not in (DataError, IntegrityError):
+ self.wrapper.errors_occurred = True
six.reraise(dj_exc_type, dj_exc_value, traceback)
def __call__(self, func):
@@ -155,6 +161,7 @@ def ensure_defaults(self, alias):
conn.setdefault('ENGINE', 'django.db.backends.dummy')
if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']:
conn['ENGINE'] = 'django.db.backends.dummy'
+ conn.setdefault('CONN_MAX_AGE', 600)
conn.setdefault('OPTIONS', {})
conn.setdefault('TIME_ZONE', 'UTC' if settings.USE_TZ else settings.TIME_ZONE)
for setting in ['NAME', 'USER', 'PASSWORD', 'HOST', 'PORT']:
View
12 django/test/client.py
@@ -18,7 +18,7 @@
from django.core.handlers.wsgi import WSGIRequest
from django.core.signals import (request_started, request_finished,
got_request_exception)
-from django.db import close_connection
+from django.db import close_old_connections
from django.http import SimpleCookie, HttpRequest, QueryDict
from django.template import TemplateDoesNotExist
from django.test import signals
@@ -78,9 +78,9 @@ def closing_iterator_wrapper(iterable, close):
for item in iterable:
yield item
finally:
- request_finished.disconnect(close_connection)
+ request_finished.disconnect(close_old_connections)
close() # will fire request_finished
- request_finished.connect(close_connection)
+ request_finished.connect(close_old_connections)
class ClientHandler(BaseHandler):
@@ -101,7 +101,9 @@ def __call__(self, environ):
if self._request_middleware is None:
self.load_middleware()
+ request_started.disconnect(close_old_connections)
request_started.send(sender=self.__class__)
+ request_started.connect(close_old_connections)
request = WSGIRequest(environ)
# sneaky little hack so that we can easily get round
# CsrfViewMiddleware. This makes life easier, and is probably
@@ -115,9 +117,9 @@ def __call__(self, environ):
response.streaming_content = closing_iterator_wrapper(
response.streaming_content, response.close)
else:
- request_finished.disconnect(close_connection)
+ request_finished.disconnect(close_old_connections)
response.close() # will fire request_finished
- request_finished.connect(close_connection)
+ request_finished.connect(close_old_connections)
return response
View
2  docs/internals/deprecation.txt
@@ -339,6 +339,8 @@ these changes.
* ``Model._meta.module_name`` was renamed to ``model_name``.
+* The private API ``django.db.close_connection`` will be removed.
+
2.0
---
View
62 docs/ref/databases.txt
@@ -11,6 +11,68 @@ This file describes some of the features that might be relevant to Django
usage. Of course, it is not intended as a replacement for server-specific
documentation or reference manuals.
+General notes
+=============
+
+.. _persistent-database-connections:
+
+Persistent connections
+----------------------
+
+.. versionadded:: 1.6
+
+Persistent connections avoid the overhead of re-establishing a connection to
+the database in each request. By default, connections are kept open for up 10
+minutes — if not specified, :setting:`CONN_MAX_AGE` defaults to 600 seconds.
+
+Django 1.5 and earlier didn't have persistent connections. To restore the
+legacy behavior of closing the connection at the end of every request, set
+:setting:`CONN_MAX_AGE` to ``0``.
+
+For unlimited persistent connections, set :setting:`CONN_MAX_AGE` to ``None``.
+
+Connection management
+~~~~~~~~~~~~~~~~~~~~~
+
+Django opens a connection to the database when it first makes a database
+query. It keeps this connection open and reuses it in subsequent requests.
+Django closes the connection once it exceeds the maximum age defined by
+:setting:`CONN_MAX_AGE` or when it isn't usable any longer.
+
+In detail, Django automatically opens a connection to the database whenever it
+needs one and doesn't have one already — either because this is the first
+connection, or because the previous connection was closed.
+
+At the beginning of each request, Django closes the connection if it has
+reached its maximum age. If your database terminates idle connections after
+some time, you should set :setting:`CONN_MAX_AGE` to a lower value, so that
+Django doesn't attempt to use a connection that has been terminated by the
+database server. (This problem may only affect very low traffic sites.)
+
+At the end of each request, Django closes the connection if it has reached its
+maximum age or if it is in an unrecoverable error state. If any database
+errors have occurred while processing the requests, Django checks whether the
+connection still works, and closes it if it doesn't. Thus, database errors
+affect at most one request; if the connection becomes unusable, the next
+request gets a fresh connection.
+
+Caveats
+~~~~~~~
+
+Since each thread maintains its own connection, your database must support at
+least as many simultaneous connections as you have worker threads.
+
+Sometimes a database won't be accessed by the majority of your views, for
+example because it's the database of an external system, or thanks to caching.
+In such cases, you should set :setting:`CONN_MAX_AGE` to a lower value, or
+even ``0``, because it doesn't make sense to maintain a connection that's
+unlikely to be reused. This will help keep the number of simultaneous
+connections to this database small.
+
+
+The development server creates a new thread for each request it handles,
+negating the effect of persistent connections.
+
.. _postgresql-notes:
PostgreSQL notes
View
13 docs/ref/settings.txt
@@ -464,6 +464,19 @@ The name of the database to use. For SQLite, it's the full path to the database
file. When specifying the path, always use forward slashes, even on Windows
(e.g. ``C:/homes/user/mysite/sqlite3.db``).
+.. setting:: CONN_MAX_AGE
+
+CONN_MAX_AGE
+~~~~~~~~~~~~
+
+.. versionadded:: 1.6
+
+Default: ``600``
+
+The lifetime of a database connection, in seconds. Use ``0`` to close database
+connections at the end of each request — Django's historical behavior — and
+``None`` for unlimited persistent connections.
+
.. setting:: OPTIONS
OPTIONS
View
21 docs/releases/1.6.txt
@@ -30,6 +30,19 @@ prevention <clickjacking-prevention>` are turned on.
If the default templates don't suit your tastes, you can use :ref:`custom
project and app templates <custom-app-and-project-templates>`.
+Persistent database connections
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Django now supports reusing the same database connection for several requests.
+This avoids the overhead of re-establishing a connection at the beginning of
+each request.
+
+By default, database connections will kept open for 10 minutes. This behavior
+is controlled by the :setting:`CONN_MAX_AGE` setting. To restore the previous
+behavior of closing the connection at the end of each request, set
+:setting:`CONN_MAX_AGE` to ``0``. See :ref:`persistent-database-connections`
+for details.
+
Time zone aware aggregation
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -136,6 +149,14 @@ Backwards incompatible changes in 1.6
* Model fields named ``hour``, ``minute`` or ``second`` may clash with the new
lookups. Append an explicit :lookup:`exact` lookup if this is an issue.
+* When Django establishes a connection to the database, it sets up appropriate
+ parameters, depending on the backend being used. Since `persistent database
+ connections <persistent-database-connections>`_ are enabled by default in
+ Django 1.6, this setup isn't repeated at every request any more. If you
+ modifiy parameters such as the connection's isolation level or time zone,
+ you should either restore Django's defaults at the end of each request, or
+ force an appropriate value at the beginning of each request.
+
* If your CSS/Javascript code used to access HTML input widgets by type, you
should review it as ``type='text'`` widgets might be now output as
``type='email'``, ``type='url'`` or ``type='number'`` depending on their
View
17 tests/handlers/tests.py
@@ -1,5 +1,6 @@
from django.core.handlers.wsgi import WSGIHandler
-from django.core import signals
+from django.core.signals import request_started, request_finished
+from django.db import close_old_connections
from django.test import RequestFactory, TestCase
from django.test.utils import override_settings
from django.utils import six
@@ -7,6 +8,12 @@
class HandlerTests(TestCase):
+ def setUp(self):
+ request_started.disconnect(close_old_connections)
+
+ def tearDown(self):
+ request_started.connect(close_old_connections)
+
# Mangle settings so the handler will fail
@override_settings(MIDDLEWARE_CLASSES=42)
def test_lock_safety(self):
@@ -35,12 +42,12 @@ class SignalsTests(TestCase):
def setUp(self):
self.signals = []
- signals.request_started.connect(self.register_started)
- signals.request_finished.connect(self.register_finished)
+ request_started.connect(self.register_started)
+ request_finished.connect(self.register_finished)
def tearDown(self):
- signals.request_started.disconnect(self.register_started)
- signals.request_finished.disconnect(self.register_finished)
+ request_started.disconnect(self.register_started)
+ request_finished.disconnect(self.register_finished)
def register_started(self, **kwargs):
self.signals.append('started')
View
6 tests/httpwrappers/tests.py
@@ -8,7 +8,7 @@
from django.core.exceptions import SuspiciousOperation
from django.core.signals import request_finished
-from django.db import close_connection
+from django.db import close_old_connections
from django.http import (QueryDict, HttpResponse, HttpResponseRedirect,
HttpResponsePermanentRedirect, HttpResponseNotAllowed,
HttpResponseNotModified, StreamingHttpResponse,
@@ -490,10 +490,10 @@ class FileCloseTests(TestCase):
def setUp(self):
# Disable the request_finished signal during this test
# to avoid interfering with the database connection.
- request_finished.disconnect(close_connection)
+ request_finished.disconnect(close_old_connections)
def tearDown(self):
- request_finished.connect(close_connection)
+ request_finished.connect(close_old_connections)
def test_response(self):
filename = os.path.join(os.path.dirname(upath(__file__)), 'abc.txt')
View
8 tests/wsgi/tests.py
@@ -2,7 +2,9 @@
from django.core.exceptions import ImproperlyConfigured
from django.core.servers.basehttp import get_internal_wsgi_application
+from django.core.signals import request_started
from django.core.wsgi import get_wsgi_application
+from django.db import close_old_connections
from django.test import TestCase
from django.test.client import RequestFactory
from django.test.utils import override_settings
@@ -12,6 +14,12 @@
class WSGITest(TestCase):
urls = "wsgi.urls"
+ def setUp(self):
+ request_started.disconnect(close_old_connections)
+
+ def tearDown(self):
+ request_started.connect(close_old_connections)
+
def test_get_wsgi_application(self):
"""
Verify that ``get_wsgi_application`` returns a functioning WSGI
Please sign in to comment.
Something went wrong with that request. Please try again.