Db scheduler: Handle database errors occuring at sync

celery · May 8, 2012 · c60a396 · c60a396
1 parent 3835ea9
commit c60a396
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 15 deletions.
diff --git a/djcelery/schedulers.py b/djcelery/schedulers.py
@@ -164,18 +164,25 @@ def reserve(self, entry):
     @transaction.commit_manually
     def sync(self):
         self.logger.debug("Writing dirty entries...")
+        _tried = set()
         try:
-            while self._dirty:
-                try:
-                    name = self._dirty.pop()
-                    self.schedule[name].save()
-                except (KeyError, ObjectDoesNotExist):
-                    pass
-        except:
-            transaction.rollback()
-            raise
-        else:
-            transaction.commit()
+            try:
+                while self._dirty:
+                    try:
+                        name = self._dirty.pop()
+                        _tried.add(name)
+                        self.schedule[name].save()
+                    except (KeyError, ObjectDoesNotExist):
+                        pass
+            except:
+                transaction.rollback()
+                raise
+            else:
+                transaction.commit()
+        except DATABASE_ERRORS, exc:
+            # retry later
+            self._dirty |= _tried
+            warn(RuntimeWarning("Database error while sync: %r" % (exc, )))
 
     def update_from_dict(self, dict_):
         s = {}

diff --git a/djcelery/utils.py b/djcelery/utils.py
@@ -10,22 +10,30 @@
 from django.db import DatabaseError
 try:
     import MySQLdb as mysql
-    _my_database_errors = (mysql.DatabaseError, )
+    _my_database_errors = (mysql.DatabaseError,
+                           mysql.InterfaceError,
+                           mysql.OperationalError)
 except ImportError:
     _my_database_errors = ()      # noqa
 try:
     import psycopg2 as pg
-    _pg_database_errors = (pg.DatabaseError, )
+    _pg_database_errors = (pg.DatabaseError,
+                           pg.InterfaceError,
+                           pg.OperationalError)
 except ImportError:
     _pg_database_errors = ()      # noqa
 try:
     import sqlite3
-    _lite_database_errors = (sqlite3.DatabaseError, )
+    _lite_database_errors = (sqlite3.DatabaseError,
+                             sqlite3.InterfaceError,
+                             sqlite3.OperationalError)
 except ImportError:
     _lite_database_errors = ()    # noqa
 try:
     import cx_Oracle as oracle
-    _oracle_database_errors = (oracle.DatabaseError, )
+    _oracle_database_errors = (oracle.DatabaseError,
+                               oracle.InterfaceError,
+                               oracle.OperationalError)
 except ImportError:
     _oracle_database_errors = ()  # noqa