Merged branch 'database-level-autocommit'.

Fixed #2227: `atomic` supports nesting.
Fixed #6623: `commit_manually` is deprecated and `atomic` doesn't suffer from this defect.
Fixed #8320: the problem wasn't identified, but the legacy transaction management is deprecated.
Fixed #10744: the problem wasn't identified, but the legacy transaction management is deprecated.
Fixed #10813: since autocommit is enabled, it isn't necessary to rollback after errors any more.
Fixed #13742: savepoints are now implemented for SQLite.
Fixed #13870: transaction management in long running processes isn't a problem any more, and it's documented.
Fixed #14970: while it digresses on transaction management, this ticket essentially asks for autocommit on PostgreSQL.
Fixed #15694: `atomic` supports nesting.
Fixed #17887: autocommit makes it impossible for a connection to stay "idle of transaction".

AUTHORS

@@ -434,6 +434,7 @@ answer newbie questions, and generally made Django that much better:
     Andreas Pelme <andreas@pelme.se>
     permonik@mesias.brnonet.cz
     peter@mymart.com
+    Christophe Pettus <xof@thebuild.com>
     pgross@thoughtworks.com
     phaedo <http://phaedo.cx/>
     phil@produxion.net

django/contrib/gis/utils/layermapping.py

@@ -555,10 +555,6 @@ def _save(feat_range=default_range, num_feat=0, num_saved=0):
                     except SystemExit:
                         raise
                     except Exception as msg:
-                        if self.transaction_mode == 'autocommit':
-                            # Rolling back the transaction so that other model saves
-                            # will work.
-                            transaction.rollback_unless_managed()
                         if strict:
                             # Bailing out if the `strict` keyword is set.
                             if not silent:

django/contrib/sessions/backends/db.py

@@ -74,7 +74,6 @@ def delete(self, session_key=None):
     @classmethod
     def clear_expired(cls):
         Session.objects.filter(expire_date__lt=timezone.now()).delete()
-        transaction.commit_unless_managed()
 
 
 # At bottom to avoid circular import

django/core/cache/backends/db.py

@@ -10,7 +10,7 @@
 
 from django.conf import settings
 from django.core.cache.backends.base import BaseCache
-from django.db import connections, router, transaction, DatabaseError
+from django.db import connections, router, DatabaseError
 from django.utils import timezone, six
 from django.utils.encoding import force_bytes
 
@@ -70,7 +70,6 @@ def get(self, key, default=None, version=None):
             cursor = connections[db].cursor()
             cursor.execute("DELETE FROM %s "
                            "WHERE cache_key = %%s" % table, [key])
-            transaction.commit_unless_managed(using=db)
             return default
         value = connections[db].ops.process_clob(row[1])
         return pickle.loads(base64.b64decode(force_bytes(value)))
@@ -124,10 +123,8 @@ def _base_set(self, mode, key, value, timeout=None):
                                [key, b64encoded, connections[db].ops.value_to_db_datetime(exp)])
         except DatabaseError:
             # To be threadsafe, updates/inserts are allowed to fail silently
-            transaction.rollback_unless_managed(using=db)
             return False
         else:
-            transaction.commit_unless_managed(using=db)
             return True
 
     def delete(self, key, version=None):
@@ -139,7 +136,6 @@ def delete(self, key, version=None):
         cursor = connections[db].cursor()
 
         cursor.execute("DELETE FROM %s WHERE cache_key = %%s" % table, [key])
-        transaction.commit_unless_managed(using=db)
 
     def has_key(self, key, version=None):
         key = self.make_key(key, version=version)
@@ -184,7 +180,6 @@ def clear(self):
         table = connections[db].ops.quote_name(self._table)
         cursor = connections[db].cursor()
         cursor.execute('DELETE FROM %s' % table)
-        transaction.commit_unless_managed(using=db)
 
 # For backwards compatibility
 class CacheClass(DatabaseCache):

django/core/handlers/base.py

@@ -6,10 +6,10 @@
 
 from django import http
 from django.conf import settings
-from django.core import exceptions
 from django.core import urlresolvers
 from django.core import signals
 from django.core.exceptions import MiddlewareNotUsed, PermissionDenied
+from django.db import connections, transaction
 from django.utils.encoding import force_text
 from django.utils.module_loading import import_by_path
 from django.utils import six
@@ -65,6 +65,13 @@ def load_middleware(self):
         # as a flag for initialization being complete.
         self._request_middleware = request_middleware
 
+    def make_view_atomic(self, view):
+        if getattr(view, 'transactions_per_request', True):
+            for db in connections.all():
+                if db.settings_dict['ATOMIC_REQUESTS']:
+                    view = transaction.atomic(using=db.alias)(view)
+        return view
+
     def get_response(self, request):
         "Returns an HttpResponse object for the given HttpRequest"
         try:
@@ -101,8 +108,9 @@ def get_response(self, request):
                             break
 
                 if response is None:
+                    wrapped_callback = self.make_view_atomic(callback)
                     try:
-                        response = callback(request, *callback_args, **callback_kwargs)
+                        response = wrapped_callback(request, *callback_args, **callback_kwargs)
                     except Exception as e:
                         # If the view raised an exception, run it through exception
                         # middleware, and if the exception middleware returns a

django/core/management/commands/createcachetable.py

@@ -53,14 +53,13 @@ def handle_label(self, tablename, **options):
         for i, line in enumerate(table_output):
             full_statement.append('    %s%s' % (line, i < len(table_output)-1 and ',' or ''))
         full_statement.append(');')
-        curs = connection.cursor()
-        try:
-            curs.execute("\n".join(full_statement))
-        except DatabaseError as e:
-            transaction.rollback_unless_managed(using=db)
-            raise CommandError(
-                "Cache table '%s' could not be created.\nThe error was: %s." %
-                    (tablename, force_text(e)))
-        for statement in index_output:
-            curs.execute(statement)
-        transaction.commit_unless_managed(using=db)
+        with transaction.commit_on_success_unless_managed():
+            curs = connection.cursor()
+            try:
+                curs.execute("\n".join(full_statement))
+            except DatabaseError as e:
+                raise CommandError(
+                    "Cache table '%s' could not be created.\nThe error was: %s." %
+                        (tablename, force_text(e)))
+            for statement in index_output:
+                curs.execute(statement)

django/core/management/commands/flush.py

@@ -57,18 +57,17 @@ def handle_noargs(self, **options):
 
         if confirm == 'yes':
             try:
-                cursor = connection.cursor()
-                for sql in sql_list:
-                    cursor.execute(sql)
+                with transaction.commit_on_success_unless_managed():
+                    cursor = connection.cursor()
+                    for sql in sql_list:
+                        cursor.execute(sql)
             except Exception as e:
-                transaction.rollback_unless_managed(using=db)
                 raise CommandError("""Database %s couldn't be flushed. Possible reasons:
   * The database isn't running or isn't configured correctly.
   * At least one of the expected database tables doesn't exist.
   * The SQL was invalid.
 Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
 The full error: %s""" % (connection.settings_dict['NAME'], e))
-            transaction.commit_unless_managed(using=db)
 
             # Emit the post sync signal. This allows individual
             # applications to respond as if the database had been

django/core/management/commands/loaddata.py

@@ -41,8 +41,6 @@ def handle(self, *fixture_labels, **options):
         self.ignore = options.get('ignore')
         self.using = options.get('database')
 
-        connection = connections[self.using]
-
         if not len(fixture_labels):
             raise CommandError(
                 "No database fixture specified. Please provide the path of at "
@@ -51,32 +49,25 @@ def handle(self, *fixture_labels, **options):
 
         self.verbosity = int(options.get('verbosity'))
 
-        # commit is a stealth option - it isn't really useful as
-        # a command line option, but it can be useful when invoking
-        # loaddata from within another script.
-        # If commit=True, loaddata will use its own transaction;
-        # if commit=False, the data load SQL will become part of
-        # the transaction in place when loaddata was invoked.
-        commit = options.get('commit', True)
+        with transaction.commit_on_success_unless_managed(using=self.using):
+            self.loaddata(fixture_labels)
+
+        # Close the DB connection -- unless we're still in a transaction. This
+        # is required as a workaround for an  edge case in MySQL: if the same
+        # connection is used to create tables, load data, and query, the query
+        # can return incorrect results. See Django #7572, MySQL #37735.
+        if transaction.get_autocommit(self.using):
+            connections[self.using].close()
+
+    def loaddata(self, fixture_labels):
+        connection = connections[self.using]
 
         # Keep a count of the installed objects and fixtures
         self.fixture_count = 0
         self.loaded_object_count = 0
         self.fixture_object_count = 0
         self.models = set()
 
-        # Get a cursor (even though we don't need one yet). This has
-        # the side effect of initializing the test database (if
-        # it isn't already initialized).
-        cursor = connection.cursor()
-
-        # Start transaction management. All fixtures are installed in a
-        # single transaction to ensure that all references are resolved.
-        if commit:
-            transaction.commit_unless_managed(using=self.using)
-            transaction.enter_transaction_management(using=self.using)
-            transaction.managed(True, using=self.using)
-
         class SingleZipReader(zipfile.ZipFile):
             def __init__(self, *args, **kwargs):
                 zipfile.ZipFile.__init__(self, *args, **kwargs)
@@ -105,26 +96,17 @@ def read(self):
 
         app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths]
 
-        try:
-            with connection.constraint_checks_disabled():
-                for fixture_label in fixture_labels:
-                    self.load_label(fixture_label, app_fixtures)
-
-            # Since we disabled constraint checks, we must manually check for
-            # any invalid keys that might have been added
-            table_names = [model._meta.db_table for model in self.models]
-            try:
-                connection.check_constraints(table_names=table_names)
-            except Exception as e:
-                e.args = ("Problem installing fixtures: %s" % e,)
-                raise
+        with connection.constraint_checks_disabled():
+            for fixture_label in fixture_labels:
+                self.load_label(fixture_label, app_fixtures)
 
-        except (SystemExit, KeyboardInterrupt):
-            raise
+        # Since we disabled constraint checks, we must manually check for
+        # any invalid keys that might have been added
+        table_names = [model._meta.db_table for model in self.models]
+        try:
+            connection.check_constraints(table_names=table_names)
         except Exception as e:
-            if commit:
-                transaction.rollback(using=self.using)
-                transaction.leave_transaction_management(using=self.using)
+            e.args = ("Problem installing fixtures: %s" % e,)
             raise
 
         # If we found even one object in a fixture, we need to reset the
@@ -137,10 +119,6 @@ def read(self):
                 for line in sequence_sql:
                     cursor.execute(line)
 
-        if commit:
-            transaction.commit(using=self.using)
-            transaction.leave_transaction_management(using=self.using)
-
         if self.verbosity >= 1:
             if self.fixture_object_count == self.loaded_object_count:
                 self.stdout.write("Installed %d object(s) from %d fixture(s)" % (
@@ -149,13 +127,6 @@ def read(self):
                 self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % (
                     self.loaded_object_count, self.fixture_object_count, self.fixture_count))
 
-        # Close the DB connection. This is required as a workaround for an
-        # edge case in MySQL: if the same connection is used to
-        # create tables, load data, and query, the query can return
-        # incorrect results. See Django #7572, MySQL #37735.
-        if commit:
-            connection.close()
-
     def load_label(self, fixture_label, app_fixtures):
 
         parts = fixture_label.split('.')

django/core/management/commands/syncdb.py

@@ -83,26 +83,25 @@ def model_installed(model):
         # Create the tables for each model
         if verbosity >= 1:
             self.stdout.write("Creating tables ...\n")
-        for app_name, model_list in manifest.items():
-            for model in model_list:
-                # Create the model's database table, if it doesn't already exist.
-                if verbosity >= 3:
-                    self.stdout.write("Processing %s.%s model\n" % (app_name, model._meta.object_name))
-                sql, references = connection.creation.sql_create_model(model, self.style, seen_models)
-                seen_models.add(model)
-                created_models.add(model)
-                for refto, refs in references.items():
-                    pending_references.setdefault(refto, []).extend(refs)
-                    if refto in seen_models:
-                        sql.extend(connection.creation.sql_for_pending_references(refto, self.style, pending_references))
-                sql.extend(connection.creation.sql_for_pending_references(model, self.style, pending_references))
-                if verbosity >= 1 and sql:
-                    self.stdout.write("Creating table %s\n" % model._meta.db_table)
-                for statement in sql:
-                    cursor.execute(statement)
-                tables.append(connection.introspection.table_name_converter(model._meta.db_table))
-
-        transaction.commit_unless_managed(using=db)
+        with transaction.commit_on_success_unless_managed(using=db):
+            for app_name, model_list in manifest.items():
+                for model in model_list:
+                    # Create the model's database table, if it doesn't already exist.
+                    if verbosity >= 3:
+                        self.stdout.write("Processing %s.%s model\n" % (app_name, model._meta.object_name))
+                    sql, references = connection.creation.sql_create_model(model, self.style, seen_models)
+                    seen_models.add(model)
+                    created_models.add(model)
+                    for refto, refs in references.items():
+                        pending_references.setdefault(refto, []).extend(refs)
+                        if refto in seen_models:
+                            sql.extend(connection.creation.sql_for_pending_references(refto, self.style, pending_references))
+                    sql.extend(connection.creation.sql_for_pending_references(model, self.style, pending_references))
+                    if verbosity >= 1 and sql:
+                        self.stdout.write("Creating table %s\n" % model._meta.db_table)
+                    for statement in sql:
+                        cursor.execute(statement)
+                    tables.append(connection.introspection.table_name_converter(model._meta.db_table))
 
         # Send the post_syncdb signal, so individual apps can do whatever they need
         # to do at this point.
@@ -122,17 +121,16 @@ def model_installed(model):
                     if custom_sql:
                         if verbosity >= 2:
                             self.stdout.write("Installing custom SQL for %s.%s model\n" % (app_name, model._meta.object_name))
-                        try:
-                            for sql in custom_sql:
-                                cursor.execute(sql)
-                        except Exception as e:
-                            self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
-                                                (app_name, model._meta.object_name, e))
-                            if show_traceback:
-                                traceback.print_exc()
-                            transaction.rollback_unless_managed(using=db)
-                        else:
-                            transaction.commit_unless_managed(using=db)
+                        with transaction.commit_on_success_unless_managed(using=db):
+                            try:
+                                for sql in custom_sql:
+                                    cursor.execute(sql)
+                            except Exception as e:
+                                self.stderr.write("Failed to install custom SQL for %s.%s model: %s\n" % \
+                                                    (app_name, model._meta.object_name, e))
+                                if show_traceback:
+                                    traceback.print_exc()
+                                raise
                     else:
                         if verbosity >= 3:
                             self.stdout.write("No custom SQL for %s.%s model\n" % (app_name, model._meta.object_name))
@@ -147,15 +145,14 @@ def model_installed(model):
                     if index_sql:
                         if verbosity >= 2:
                             self.stdout.write("Installing index for %s.%s model\n" % (app_name, model._meta.object_name))
-                        try:
-                            for sql in index_sql:
-                                cursor.execute(sql)
-                        except Exception as e:
-                            self.stderr.write("Failed to install index for %s.%s model: %s\n" % \
-                                                (app_name, model._meta.object_name, e))
-                            transaction.rollback_unless_managed(using=db)
-                        else:
-                            transaction.commit_unless_managed(using=db)
+                        with transaction.commit_on_success_unless_managed(using=db):
+                            try:
+                                for sql in index_sql:
+                                    cursor.execute(sql)
+                            except Exception as e:
+                                self.stderr.write("Failed to install index for %s.%s model: %s\n" % \
+                                                    (app_name, model._meta.object_name, e))
+                                raise
 
         # Load initial_data fixtures (unless that has been disabled)
         if load_initial_data:

django/db/__init__.py

@@ -70,21 +70,11 @@ def reset_queries(**kwargs):
 # their lifetime. NB: abort() doesn't do anything outside of a transaction.
 def close_old_connections(**kwargs):
     for conn in connections.all():
+        # Remove this when the legacy transaction management goes away.
         try:
             conn.abort()
         except DatabaseError:
             pass
         conn.close_if_unusable_or_obsolete()
 signals.request_started.connect(close_old_connections)
 signals.request_finished.connect(close_old_connections)
-
-# Register an event that rolls back the connections
-# when a Django request has an exception.
-def _rollback_on_exception(**kwargs):
-    from django.db import transaction
-    for conn in connections:
-        try:
-            transaction.rollback_unless_managed(using=conn)
-        except DatabaseError:
-            pass
-signals.got_request_exception.connect(_rollback_on_exception)