Permalink
Browse files

Merged branch 'database-level-autocommit'.

Fixed #2227: `atomic` supports nesting.
Fixed #6623: `commit_manually` is deprecated and `atomic` doesn't suffer from this defect.
Fixed #8320: the problem wasn't identified, but the legacy transaction management is deprecated.
Fixed #10744: the problem wasn't identified, but the legacy transaction management is deprecated.
Fixed #10813: since autocommit is enabled, it isn't necessary to rollback after errors any more.
Fixed #13742: savepoints are now implemented for SQLite.
Fixed #13870: transaction management in long running processes isn't a problem any more, and it's documented.
Fixed #14970: while it digresses on transaction management, this ticket essentially asks for autocommit on PostgreSQL.
Fixed #15694: `atomic` supports nesting.
Fixed #17887: autocommit makes it impossible for a connection to stay "idle of transaction".
  • Loading branch information...
2 parents 9cec689 + e654180 commit 14cddf51c5f001bb426ce7f7a83fdc52c8d8aee9 @aaugustin aaugustin committed Mar 11, 2013
Showing with 1,747 additions and 936 deletions.
  1. +1 −0 AUTHORS
  2. +0 −4 django/contrib/gis/utils/layermapping.py
  3. +0 −1 django/contrib/sessions/backends/db.py
  4. +1 −6 django/core/cache/backends/db.py
  5. +10 −2 django/core/handlers/base.py
  6. +10 −11 django/core/management/commands/createcachetable.py
  7. +4 −5 django/core/management/commands/flush.py
  8. +21 −50 django/core/management/commands/loaddata.py
  9. +37 −40 django/core/management/commands/syncdb.py
  10. +1 −11 django/db/__init__.py
  11. +140 −107 django/db/backends/__init__.py
  12. +6 −15 django/db/backends/creation.py
  13. +1 −4 django/db/backends/dummy/base.py
  14. +3 −9 django/db/backends/mysql/base.py
  15. +3 −0 django/db/backends/oracle/base.py
  16. +0 −3 django/db/backends/oracle/creation.py
  17. +26 −47 django/db/backends/postgresql_psycopg2/base.py
  18. +0 −11 django/db/backends/postgresql_psycopg2/creation.py
  19. +0 −9 django/db/backends/postgresql_psycopg2/operations.py
  20. +32 −0 django/db/backends/sqlite3/base.py
  21. +0 −3 django/db/backends/sqlite3/creation.py
  22. +42 −42 django/db/models/base.py
  23. +32 −52 django/db/models/deletion.py
  24. +2 −26 django/db/models/query.py
  25. +251 −36 django/db/transaction.py
  26. +8 −0 django/db/utils.py
  27. +13 −3 django/middleware/transaction.py
  28. +10 −26 django/test/testcases.py
  29. +17 −2 docs/internals/deprecation.txt
  30. +13 −46 docs/ref/databases.txt
  31. +4 −0 docs/ref/middleware.txt
  32. +2 −2 docs/ref/request-response.txt
  33. +30 −0 docs/ref/settings.txt
  34. +2 −4 docs/releases/1.3-alpha-1.txt
  35. +2 −4 docs/releases/1.3.txt
  36. +37 −0 docs/releases/1.6.txt
  37. +26 −35 docs/topics/db/sql.txt
  38. +525 −219 docs/topics/db/transactions.txt
  39. +14 −5 tests/backends/tests.py
  40. +1 −3 tests/delete_regress/tests.py
  41. +9 −2 tests/fixtures_model_package/tests.py
  42. +5 −2 tests/fixtures_regress/tests.py
  43. +27 −3 tests/handlers/tests.py
  44. +6 −3 tests/handlers/urls.py
  45. +17 −0 tests/handlers/views.py
  46. +7 −15 tests/middleware/tests.py
  47. +0 −2 tests/requests/tests.py
  48. +0 −3 tests/select_for_update/tests.py
  49. +0 −1 tests/serializers/tests.py
  50. +1 −1 tests/transactions/models.py
  51. +311 −3 tests/transactions/tests.py
  52. +37 −58 tests/transactions_regress/tests.py
View
@@ -434,6 +434,7 @@ answer newbie questions, and generally made Django that much better:
Andreas Pelme <andreas@pelme.se>
permonik@mesias.brnonet.cz
peter@mymart.com
+ Christophe Pettus <xof@thebuild.com>
pgross@thoughtworks.com
phaedo <http://phaedo.cx/>
phil@produxion.net
@@ -555,10 +555,6 @@ def _save(feat_range=default_range, num_feat=0, num_saved=0):
except SystemExit:
raise
except Exception as msg:
- if self.transaction_mode == 'autocommit':
- # Rolling back the transaction so that other model saves
- # will work.
- transaction.rollback_unless_managed()
if strict:
# Bailing out if the `strict` keyword is set.
if not silent:
@@ -74,7 +74,6 @@ def delete(self, session_key=None):
@classmethod
def clear_expired(cls):
Session.objects.filter(expire_date__lt=timezone.now()).delete()
- transaction.commit_unless_managed()
# At bottom to avoid circular import
@@ -10,7 +10,7 @@
from django.conf import settings
from django.core.cache.backends.base import BaseCache
-from django.db import connections, router, transaction, DatabaseError
+from django.db import connections, router, DatabaseError
from django.utils import timezone, six
from django.utils.encoding import force_bytes
@@ -70,7 +70,6 @@ def get(self, key, default=None, version=None):
cursor = connections[db].cursor()
cursor.execute("DELETE FROM %s "
"WHERE cache_key = %%s" % table, [key])
- transaction.commit_unless_managed(using=db)
return default
value = connections[db].ops.process_clob(row[1])
return pickle.loads(base64.b64decode(force_bytes(value)))
@@ -124,10 +123,8 @@ def _base_set(self, mode, key, value, timeout=None):
[key, b64encoded, connections[db].ops.value_to_db_datetime(exp)])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
- transaction.rollback_unless_managed(using=db)
return False
else:
- transaction.commit_unless_managed(using=db)
return True
def delete(self, key, version=None):
@@ -139,7 +136,6 @@ def delete(self, key, version=None):
cursor = connections[db].cursor()
cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % table, [key])
- transaction.commit_unless_managed(using=db)
def has_key(self, key, version=None):
key = self.make_key(key, version=version)
@@ -184,7 +180,6 @@ def clear(self):
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute('DELETE FROM %s' % table)
- transaction.commit_unless_managed(using=db)
# For backwards compatibility
class CacheClass(DatabaseCache):
@@ -6,10 +6,10 @@
from django import http
from django.conf import settings
-from django.core import exceptions
from django.core import urlresolvers
from django.core import signals
from django.core.exceptions import MiddlewareNotUsed, PermissionDenied
+from django.db import connections, transaction
from django.utils.encoding import force_text
from django.utils.module_loading import import_by_path
from django.utils import six
@@ -65,6 +65,13 @@ def load_middleware(self):
# as a flag for initialization being complete.
self._request_middleware = request_middleware
+ def make_view_atomic(self, view):
+ if getattr(view, 'transactions_per_request', True):
+ for db in connections.all():
+ if db.settings_dict['ATOMIC_REQUESTS']:
+ view = transaction.atomic(using=db.alias)(view)
+ return view
+
def get_response(self, request):
"Returns an HttpResponse object for the given HttpRequest"
try:
@@ -101,8 +108,9 @@ def get_response(self, request):
break
if response is None:
+ wrapped_callback = self.make_view_atomic(callback)
try:
- response = callback(request, *callback_args, **callback_kwargs)
+ response = wrapped_callback(request, *callback_args, **callback_kwargs)
except Exception as e:
# If the view raised an exception, run it through exception
# middleware, and if the exception middleware returns a
@@ -53,14 +53,13 @@ def handle_label(self, tablename, **options):
for i, line in enumerate(table_output):
full_statement.append(' %s%s' % (line, i < len(table_output)-1 and ',' or ''))
full_statement.append(');')
- curs = connection.cursor()
- try:
- curs.execute("\n".join(full_statement))
- except DatabaseError as e:
- transaction.rollback_unless_managed(using=db)
- raise CommandError(
- "Cache table '%s' could not be created.\nThe error was: %s." %
- (tablename, force_text(e)))
- for statement in index_output:
- curs.execute(statement)
- transaction.commit_unless_managed(using=db)
+ with transaction.commit_on_success_unless_managed():
+ curs = connection.cursor()
+ try:
+ curs.execute("\n".join(full_statement))
+ except DatabaseError as e:
+ raise CommandError(
+ "Cache table '%s' could not be created.\nThe error was: %s." %
+ (tablename, force_text(e)))
+ for statement in index_output:
+ curs.execute(statement)
@@ -57,18 +57,17 @@ def handle_noargs(self, **options):
if confirm == 'yes':
try:
- cursor = connection.cursor()
- for sql in sql_list:
- cursor.execute(sql)
+ with transaction.commit_on_success_unless_managed():
+ cursor = connection.cursor()
+ for sql in sql_list:
+ cursor.execute(sql)
except Exception as e:
- transaction.rollback_unless_managed(using=db)
raise CommandError("""Database %s couldn't be flushed. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the expected database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
The full error: %s""" % (connection.settings_dict['NAME'], e))
- transaction.commit_unless_managed(using=db)
# Emit the post sync signal. This allows individual
# applications to respond as if the database had been
@@ -41,8 +41,6 @@ def handle(self, *fixture_labels, **options):
self.ignore = options.get('ignore')
self.using = options.get('database')
- connection = connections[self.using]
-
if not len(fixture_labels):
raise CommandError(
"No database fixture specified. Please provide the path of at "
@@ -51,32 +49,25 @@ def handle(self, *fixture_labels, **options):
self.verbosity = int(options.get('verbosity'))
- # commit is a stealth option - it isn't really useful as
- # a command line option, but it can be useful when invoking
- # loaddata from within another script.
- # If commit=True, loaddata will use its own transaction;
- # if commit=False, the data load SQL will become part of
- # the transaction in place when loaddata was invoked.
- commit = options.get('commit', True)
+ with transaction.commit_on_success_unless_managed(using=self.using):
+ self.loaddata(fixture_labels)
+
+ # Close the DB connection -- unless we're still in a transaction. This
+ # is required as a workaround for an edge case in MySQL: if the same
+ # connection is used to create tables, load data, and query, the query
+ # can return incorrect results. See Django #7572, MySQL #37735.
+ if transaction.get_autocommit(self.using):
+ connections[self.using].close()
+
+ def loaddata(self, fixture_labels):
+ connection = connections[self.using]
# Keep a count of the installed objects and fixtures
self.fixture_count = 0
self.loaded_object_count = 0
self.fixture_object_count = 0
self.models = set()
- # Get a cursor (even though we don't need one yet). This has
- # the side effect of initializing the test database (if
- # it isn't already initialized).
- cursor = connection.cursor()
-
- # Start transaction management. All fixtures are installed in a
- # single transaction to ensure that all references are resolved.
- if commit:
- transaction.commit_unless_managed(using=self.using)
- transaction.enter_transaction_management(using=self.using)
- transaction.managed(True, using=self.using)
-
class SingleZipReader(zipfile.ZipFile):
def __init__(self, *args, **kwargs):
zipfile.ZipFile.__init__(self, *args, **kwargs)
@@ -105,26 +96,17 @@ def read(self):
app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths]
- try:
- with connection.constraint_checks_disabled():
- for fixture_label in fixture_labels:
- self.load_label(fixture_label, app_fixtures)
-
- # Since we disabled constraint checks, we must manually check for
- # any invalid keys that might have been added
- table_names = [model._meta.db_table for model in self.models]
- try:
- connection.check_constraints(table_names=table_names)
- except Exception as e:
- e.args = ("Problem installing fixtures: %s" % e,)
- raise
+ with connection.constraint_checks_disabled():
+ for fixture_label in fixture_labels:
+ self.load_label(fixture_label, app_fixtures)
- except (SystemExit, KeyboardInterrupt):
- raise
+ # Since we disabled constraint checks, we must manually check for
+ # any invalid keys that might have been added
+ table_names = [model._meta.db_table for model in self.models]
+ try:
+ connection.check_constraints(table_names=table_names)
except Exception as e:
- if commit:
- transaction.rollback(using=self.using)
- transaction.leave_transaction_management(using=self.using)
+ e.args = ("Problem installing fixtures: %s" % e,)
raise
# If we found even one object in a fixture, we need to reset the
@@ -137,10 +119,6 @@ def read(self):
for line in sequence_sql:
cursor.execute(line)
- if commit:
- transaction.commit(using=self.using)
- transaction.leave_transaction_management(using=self.using)
-
if self.verbosity >= 1:
if self.fixture_object_count == self.loaded_object_count:
self.stdout.write("Installed %d object(s) from %d fixture(s)" % (
@@ -149,13 +127,6 @@ def read(self):
self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % (
self.loaded_object_count, self.fixture_object_count, self.fixture_count))
- # Close the DB connection. This is required as a workaround for an
- # edge case in MySQL: if the same connection is used to
- # create tables, load data, and query, the query can return
- # incorrect results. See Django #7572, MySQL #37735.
- if commit:
- connection.close()
-
def load_label(self, fixture_label, app_fixtures):
parts = fixture_label.split('.')
@@ -83,26 +83,25 @@ def model_installed(model):
# Create the tables for each model
if verbosity >= 1:
self.stdout.write("Creating tables ...\n")
- for app_name, model_list in manifest.items():
- for model in model_list:
- # Create the model's database table, if it doesn't already exist.
- if verbosity >= 3:
- self.stdout.write("Processing %s.%s model\n" % (app_name, model._meta.object_name))
- sql, references = connection.creation.sql_create_model(model, self.style, seen_models)
- seen_models.add(model)
- created_models.add(model)
- for refto, refs in references.items():
- pending_references.setdefault(refto, []).extend(refs)
- if refto in seen_models:
- sql.extend(connection.creation.sql_for_pending_references(refto, self.style, pending_references))
- sql.extend(connection.creation.sql_for_pending_references(model, self.style, pending_references))
- if verbosity >= 1 and sql:
- self.stdout.write("Creating table %s\n" % model._meta.db_table)
- for statement in sql:
- cursor.execute(statement)
- tables.append(connection.introspection.table_name_converter(model._meta.db_table))
-
- transaction.commit_unless_managed(using=db)
+ with transaction.commit_on_success_unless_managed(using=db):
+ for app_name, model_list in manifest.items():
+ for model in model_list:
+ # Create the model's database table, if it doesn't already exist.
+ if verbosity >= 3:
+ self.stdout.write("Processing %s.%s model\n" % (app_name, model._meta.object_name))
+ sql, references = connection.creation.sql_create_model(model, self.style, seen_models)
+ seen_models.add(model)
+ created_models.add(model)
+ for refto, refs in references.items():
+ pending_references.setdefault(refto, []).extend(refs)
+ if refto in seen_models:
+ sql.extend(connection.creation.sql_for_pending_references(refto, self.style, pending_references))
+ sql.extend(connection.creation.sql_for_pending_references(model, self.style, pending_references))
+ if verbosity >= 1 and sql:
+ self.stdout.write("Creating table %s\n" % model._meta.db_table)
+ for statement in sql:
+ cursor.execute(statement)
+ tables.append(connection.introspection.table_name_converter(model._meta.db_table))
# Send the post_syncdb signal, so individual apps can do whatever they need
# to do at this point.
@@ -122,17 +121,16 @@ def model_installed(model):
if custom_sql:
if verbosity >= 2:
self.stdout.write("Installing custom SQL for %s.%s model\n" % (app_name, model._meta.object_name))
- try:
- for sql in custom_sql:
- cursor.execute(sql)
- except Exception as e:
- self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
- (app_name, model._meta.object_name, e))
- if show_traceback:
- traceback.print_exc()
- transaction.rollback_unless_managed(using=db)
- else:
- transaction.commit_unless_managed(using=db)
+ with transaction.commit_on_success_unless_managed(using=db):
+ try:
+ for sql in custom_sql:
+ cursor.execute(sql)
+ except Exception as e:
+ self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
+ (app_name, model._meta.object_name, e))
+ if show_traceback:
+ traceback.print_exc()
+ raise
else:
if verbosity >= 3:
self.stdout.write("No custom SQL for %s.%s model\n" % (app_name, model._meta.object_name))
@@ -147,15 +145,14 @@ def model_installed(model):
if index_sql:
if verbosity >= 2:
self.stdout.write("Installing index for %s.%s model\n" % (app_name, model._meta.object_name))
- try:
- for sql in index_sql:
- cursor.execute(sql)
- except Exception as e:
- self.stderr.write("Failed to install index for %s.%s model: %s\n" % \
- (app_name, model._meta.object_name, e))
- transaction.rollback_unless_managed(using=db)
- else:
- transaction.commit_unless_managed(using=db)
+ with transaction.commit_on_success_unless_managed(using=db):
+ try:
+ for sql in index_sql:
+ cursor.execute(sql)
+ except Exception as e:
+ self.stderr.write("Failed to install index for %s.%s model: %s\n" % \
+ (app_name, model._meta.object_name, e))
+ raise
# Load initial_data fixtures (unless that has been disabled)
if load_initial_data:
View
@@ -70,21 +70,11 @@ def reset_queries(**kwargs):
# their lifetime. NB: abort() doesn't do anything outside of a transaction.
def close_old_connections(**kwargs):
for conn in connections.all():
+ # Remove this when the legacy transaction management goes away.
try:
conn.abort()
except DatabaseError:
pass
conn.close_if_unusable_or_obsolete()
signals.request_started.connect(close_old_connections)
signals.request_finished.connect(close_old_connections)
-
-# Register an event that rolls back the connections
-# when a Django request has an exception.
-def _rollback_on_exception(**kwargs):
- from django.db import transaction
- for conn in connections:
- try:
- transaction.rollback_unless_managed(using=conn)
- except DatabaseError:
- pass
-signals.got_request_exception.connect(_rollback_on_exception)
Oops, something went wrong.

0 comments on commit 14cddf5

Please sign in to comment.