Skip to content

Commit

Permalink
feat(db): automatically retry txns on deadlocks
Browse files Browse the repository at this point in the history
This commit implements automatic retrying of transactions when a
transaction deadlock occurs.  This allows the server to handle up to 20
simultaneous patient registrations!  The integration tests have been
updated accordingly to stress the server.

The number of retries is currently 3, but is configurable via the
MAX_TRANSACTION_DEADLOCK_RESTARTS constant in the transaction library.

Closes #620.
  • Loading branch information
Jonathan Niles committed Aug 4, 2016
1 parent f1d629c commit 351e331
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 4 deletions.
27 changes: 26 additions & 1 deletion server/lib/db/transaction.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
const q = require('q');
const winston = require('winston');

/** @const the number of times a transaction is restarted in case of deadlock*/
const MAX_TRANSACTION_DEADLOCK_RESTARTS = 3;

// Uses an already existing connection to query the database, returning a promise
function queryConnection(connection, sql, params) {
const deferred = q.defer();
Expand Down Expand Up @@ -52,6 +55,7 @@ class Transaction {
constructor(db) {
this.queries = [];
this.db = db;
this.restarts = 0;
}

/**
Expand Down Expand Up @@ -124,9 +128,30 @@ class Transaction {
// individual query did not work - rollback transaction
connection.rollback(() => {
connection.destroy();
winston.debug(`[Transaction] Rollback due to : ${error}`);
winston.warn(
`[Transaction] Encountered error ${error.code}. Rolling transaction back and recoverying database connections.`
);
});

// restart transactions a set number of times if the error is due to table deadlocks
if (error.code === 'ER_LOCK_DEADLOCK' && this.restarts++ < MAX_TRANSACTION_DEADLOCK_RESTARTS) {
winston.warn(
`[Transaction] Transacton deadlock discovered. Attempting ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts.`
);

// restart transaction
return this.execute()
.then(results => deferred.resolve(results))
.catch(err => deferred.reject(err));
}

// if we get here, all attempted restarts failed. Report an error in case tables are permanently locked.
if (error.code === 'ER_LOCK_DEADLOCK') {
winston.error(
`[Transaction] Unrecoverable deadlock error. Completed ${this.restarts} / ${MAX_TRANSACTION_DEADLOCK_RESTARTS} restarts.`
);
}

return deferred.reject(error);
});
});
Expand Down
2 changes: 2 additions & 0 deletions server/lib/helpers/translate.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
'use strict';

const en = require('../../../client/i18n/en.json');
const fr = require('../../../client/i18n/fr.json');

Expand Down
1 change: 1 addition & 0 deletions server/lib/renderers/html.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
* @requires server/lib/template
*/
'use strict';

const exhbs = require('express-handlebars');

const hbs = require('../template');
Expand Down
2 changes: 2 additions & 0 deletions server/lib/renderers/json.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
*
* @module lib/renderers/json
*/
'use strict';

var q = require('q');

exports.render = renderJSON;
Expand Down
5 changes: 2 additions & 3 deletions server/test/api/patients.js
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ describe('(/patients) Patients', function () {
// var NUMBER_OF_PATIENTS = 200;
// var timeoutInterval = 30;

var NUMBER_OF_PATIENTS = 2;
var NUMBER_OF_PATIENTS = 7;
var timeoutInterval = 0;

var timeout = 0;
Expand Down Expand Up @@ -294,7 +294,7 @@ describe('(/patients) Patients', function () {

// TODO get information on the registered patient - ensure details route is correct
function delayPatientRequest(timeout, hospitalNo) {
var deferred = q.defer();
const deferred = q.defer();

setTimeout(function () {

Expand All @@ -304,7 +304,6 @@ describe('(/patients) Patients', function () {
agent.post('/patients')
.send(simultaneousRequest)
.then(function (res) {
console.log('res:', res.body);
deferred.resolve(res);
})
.catch(function (error) {
Expand Down

0 comments on commit 351e331

Please sign in to comment.