feat(db): automatically retry txns on deadlocks

This commit implements automatic retrying of transactions when a
transaction deadlock occurs.  This allows the server to handle up to 20
simultaneous patient registrations!  The integration tests have been
updated accordingly to stress the server.

The number of retries is currently 3, but is configurable via the
MAX_TRANSACTION_DEADLOCK_RESTARTS constant in the transaction library.

Closes IMA-WorldHealth#620.

server/lib/db/transaction.js

@@ -3,6 +3,9 @@
 const q = require('q');
 const winston = require('winston');
 
+/** @const the number of times a transaction is restarted in case of deadlock*/
+const MAX_TRANSACTION_DEADLOCK_RESTARTS = 3;
+
 // Uses an already existing connection to query the database, returning a promise
 function queryConnection(connection, sql, params) {
   const deferred = q.defer();
@@ -52,6 +55,7 @@ class Transaction {
   constructor(db) {
     this.queries = [];
     this.db = db;
+    this.restarts = 0;
   }
 
   /**
@@ -124,9 +128,30 @@ class Transaction {
             // individual query did not work - rollback transaction
             connection.rollback(() => {
               connection.destroy();
-              winston.debug(`[Transaction] Rollback due to : ${error}`);
+              winston.warn(
+                `[Transaction] Encountered error ${error.code}.  Rolling transaction back and recoverying database connections.`
+              );
             });
 
+            // restart transactions a set number of times if the error is due to table deadlocks
+            if (error.code === 'ER_LOCK_DEADLOCK' && this.restarts++ < MAX_TRANSACTION_DEADLOCK_RESTARTS) {
+              winston.warn(
+                `[Transaction] Transacton deadlock discovered. Attempting ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts.`
+              );
+
+              // restart transaction
+              return this.execute()
+                .then(results => deferred.resolve(results))
+                .catch(err => deferred.reject(err));
+            }
+
+            // if we get here, all attempted restarts failed.  Report an error in case tables are permanently locked.
+            if (error.code === 'ER_LOCK_DEADLOCK') {
+              winston.error(
+                `[Transaction] Unrecoverable deadlock error. Completed ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts.`
+              );
+            }
+
             return deferred.reject(error);
           });
       });

server/test/api/patients.js

@@ -240,7 +240,7 @@ describe('(/patients) Patients', function () {
     // var NUMBER_OF_PATIENTS = 200;
     // var timeoutInterval = 30;
 
-    var NUMBER_OF_PATIENTS = 2;
+    var NUMBER_OF_PATIENTS = 20;
     var timeoutInterval = 0;
 
     var timeout = 0;
@@ -294,7 +294,7 @@ describe('(/patients) Patients', function () {
 
   // TODO get information on the registered patient - ensure details route is correct
   function delayPatientRequest(timeout, hospitalNo) {
-    var deferred = q.defer();
+    const deferred = q.defer();
 
     setTimeout(function () {
 
@@ -304,7 +304,6 @@ describe('(/patients) Patients', function () {
       agent.post('/patients')
         .send(simultaneousRequest)
         .then(function (res) {
-          console.log('res:', res.body);
           deferred.resolve(res);
         })
         .catch(function (error) {