fix(db): set production deadlock delay

This commit fixes the transaction library by ensuring that:
 1) Deadlocks are asynchronously handled.
 2) MAX_TRANSACTION_DEADLOCK_RESTARTS is now 10.

It also makes all deadlock errors actual errors and includes a timestamp
to help identify when they occurred.

server/lib/db/transaction.js

@@ -4,7 +4,10 @@ const q = require('q');
 const winston = require('winston');
 
 /** @const the number of times a transaction is restarted in case of deadlock*/
-const MAX_TRANSACTION_DEADLOCK_RESTARTS = 3;
+const MAX_TRANSACTION_DEADLOCK_RESTARTS = 10;
+
+/** @const the number of milliseconds delayed before restarting the transaction */
+const TRANSACTION_DEADLOCK_RESTART_DELAY = 100;
 
 // Uses an already existing connection to query the database, returning a promise
 function queryConnection(connection, sql, params) {
@@ -128,27 +131,35 @@ class Transaction {
             // individual query did not work - rollback transaction
             connection.rollback(() => {
               connection.destroy();
-              winston.warn(
+              winston.error(
                 `[Transaction] Encountered error ${error.code}.  Rolling transaction back and recoverying database connections.`
               );
             });
 
             // restart transactions a set number of times if the error is due to table deadlocks
             if (error.code === 'ER_LOCK_DEADLOCK' && this.restarts++ < MAX_TRANSACTION_DEADLOCK_RESTARTS) {
-              winston.warn(
-                `[Transaction] Transacton deadlock discovered. Attempting ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts.`
+              winston.error(
+                `[Transaction] Transacton deadlock discovered. Attempting ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts. [${ new Date().toLocaleString() }]`
               );
 
-              // restart transaction
-              return this.execute()
+              // set up a promise to delay the transaction restart
+              const delay = q.defer();
+
+              // restart transaction after a delay
+              setTimeout(() => {
+                delay.resolve(this.execute());
+              }, TRANSACTION_DEADLOCK_RESTART_DELAY);
+
+              // return the promise
+              return delay.promise
                 .then(results => deferred.resolve(results))
                 .catch(err => deferred.reject(err));
             }
 
             // if we get here, all attempted restarts failed.  Report an error in case tables are permanently locked.
             if (error.code === 'ER_LOCK_DEADLOCK') {
               winston.error(
-                `[Transaction] Unrecoverable deadlock error. Completed ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts.`
+                `[Transaction] Unrecoverable deadlock error. Completed ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts. [${ new Date().toLocaleString() }]`
               );
             }
 

test/integration/patients.js

@@ -59,7 +59,7 @@ describe('(/patients) Patients', function () {
   };
 
   var simultaneousPatient = {
-    first_name:          'Simultaneous Patient Last',
+    display_name:        'Simultaneous Patient Mocks',
     dob:                 new Date('1993-06-01'),
     current_location_id: '1f162a10-9f67-4788-9eff-c1fea42fcc9b',
     origin_location_id:  '1f162a10-9f67-4788-9eff-c1fea42fcc9b',
@@ -234,17 +234,17 @@ describe('(/patients) Patients', function () {
     // Custom timeout
     this.timeout(30000);
 
-    var patientQuery = [];
+    let patientQuery = [];
 
     // Extreme case
     // var NUMBER_OF_PATIENTS = 200;
     // var timeoutInterval = 30;
 
-    var NUMBER_OF_PATIENTS = 7;
-    var timeoutInterval = 0;
+    let NUMBER_OF_PATIENTS = 100;
+    let timeoutInterval = 0;
 
-    var timeout = 0;
-    var baseHospitalNo = 1000;
+    let timeout = 0;
+    let baseHospitalNo = 300;
 
     // Setup all patient write requests
     for (var i = 0; i < NUMBER_OF_PATIENTS; i++) {
@@ -303,7 +303,10 @@ describe('(/patients) Patients', function () {
     setTimeout(function () {
 
       simultaneousRequest.medical.hospital_no = hospitalNo;
-      simultaneousRequest.medical.display_name += hospitalNo;
+
+      let name = 'Patient ';
+      let randomSuffix = (Math.random()*1.31).toString().slice(2, 10);
+      simultaneousRequest.medical.display_name = name + randomSuffix;
 
       agent.post('/patients')
         .send(simultaneousRequest)