Replication (#242)

* Using sequelize replication option to route read queries to read-only databases

* Typo fixes in the comments

config.js

@@ -50,6 +50,15 @@ const rateWindow = pe.RATE_WINDOW;
 const endpointToLimit = pe.ENDPOINT_TO_LIMIT;
 const httpMethodToLimit = pe.HTTP_METHOD_TO_LIMIT;
 
+/*
+ * name of the environment variable containing the read-only
+ * database names as CSV
+ */
+const replicaConfigLabel = 'REPLICAS';
+
+// an array of read-only data base URLs
+const readReplicas = configUtil.getReadReplicas(pe, replicaConfigLabel);
+
 const DEFAULT_JOB_QUEUE_TTL_SECONDS = 3600;
 
 /*
@@ -194,4 +203,5 @@ module.exports = {
   httpMethodToLimit,
   prioritizeJobsFrom,
   deprioritizeJobsFrom,
+  readReplicas,
 };

config/configUtil.js

@@ -63,7 +63,30 @@ function csvToArray(str) {
   return [];
 } // csvToArray
 
+/**
+ * Returns all the DB URLs of the configured read-only replicas as an array.
+ * @param  {Object} pe - Node process environment variable(process.env)
+ * @param  {String} replicaLabel - "Config variable name" that contains that
+ * name of the databases configured as read-only replicas
+ * @returns {Array} an array of all the dburls configured as read-only replicas.
+ */
+function getReadReplicas(pe, replicaLabel) {
+  let replicas;
+  if (pe[replicaLabel]) {
+    const replicaList = csvToArray(pe[replicaLabel]);
+    replicas = [];
+    replicaList.forEach((replica) => {
+      if (pe[replica]) {
+        replicas.push(pe[replica]);
+      }
+    });
+  }
+
+  return replicas;
+} // getReadReplicas
+
 module.exports = {
   csvToArray,
   parseIPlist,
+  getReadReplicas,
 };

db/utils.js

@@ -20,10 +20,8 @@ const Sequelize = require('sequelize');
 require('sequelize-hierarchy')(Sequelize);
 const conf = require('../config');
 const env = conf.environment[conf.nodeEnv];
-const seq = new Sequelize(env.dbUrl, {
-  logging: env.dbLogging,
-});
 const DB_URL = env.dbUrl;
+
 const SQL_DROP_SEQUELIZE_META = 'DROP TABLE IF EXISTS public."SequelizeMeta"';
 const SQL_INSERT_SEQUELIZE_META = 'INSERT INTO "SequelizeMeta"(name) VALUES';
 const SQL_CREATE_SEQUELIZE_META = 'CREATE TABLE public."SequelizeMeta" ' +
@@ -52,6 +50,97 @@ function dbConfigObjectFromDbURL(dbUrl) {
   };
 } // dbConfigObjectFromDbURL
 
+/**
+ * Returns an array of database config object, that is mapped to the read
+ * property of the sequelize replication option.
+ * @param  {Array} readReplicas - An array of database urls
+ * @returns {Array} - any array of database config object parsed from
+ * dbConfigObjectFromDbURL function
+ */
+function getReadOnlyDBConfig(readReplicas) {
+  let readConfig;
+  if (Array.isArray(readReplicas) && readReplicas.length) {
+    readConfig = [];
+    readReplicas.forEach((replicaUrl) => {
+      const dbConfObj = dbConfigObjectFromDbURL(replicaUrl);
+      readConfig.push({
+        host: dbConfObj.host,
+        port: dbConfObj.port,
+        username: dbConfObj.user,
+        password: dbConfObj.password,
+      });
+    });
+  }
+
+  return readConfig;
+} // getReadOnlyDBConfig
+
+/**
+ * Returns the configuration object for the "replication" option in sequelize.
+ * The primary database config object is mapped to the write property of the
+ * sequelize replication option. If any read-only replicas are configured, they
+ * are mapped to the read property of the sequelize replication option. For,
+ * example when the read-only replica is configured, the returned object is
+ * of the form
+ * {write : {
+ *   'host': host,
+ *   'port': port,
+ *   'username': username,
+ *   'password': password,
+ *   },
+ *  read: [{
+ *   'host': host,
+ *   'port': port,
+ *   'username': username,
+ *   'password': password,
+ *   }]
+ * }
+ * @param  {Object}primaryDBConfObj  - The primary database config object
+ * parsed from dbConfigObjectFromDbURL function
+ * @returns {Object} - an object of the shape shown above
+ */
+function getDBReplicationObject(primaryDBConfObj) {
+  const repObj = {};
+  repObj.write = {
+    host: primaryDBConfObj.host,
+    port: primaryDBConfObj.port,
+    username: primaryDBConfObj.user,
+    password: primaryDBConfObj.password,
+  };
+  const readReplicaObject = getReadOnlyDBConfig(conf.readReplicas);
+  if (Array.isArray(readReplicaObject) && readReplicaObject.length) {
+    repObj.read = readReplicaObject;
+  }
+
+  return repObj;
+} // getDBReplicationObject
+
+// this is the master database where the writes happen
+const primaryDb = dbConfigObjectFromDbURL();
+
+// create the sequelize object from the constructor.
+let seq;
+if (conf.readReplicas) {
+
+  /*
+   * The sequelize object is constructed this way if read-only replicas are
+   * configured.
+   * This usage is of the form new Sequelize('database',
+   * 'username' 'password , options).
+   * The username and password are passed to the constructor through the
+   * replication property of options.
+   */
+  seq = new Sequelize(primaryDb.name, null, null, {
+    dialect: env.dialect,
+    replication: getDBReplicationObject(primaryDb),
+    logging: env.dbLogging,
+  });
+} else {
+  seq = new Sequelize(env.dbUrl, {
+    logging: env.dbLogging,
+  });
+}
+
 /**
  * A console logging wrapper for stuff running from the command line.
  *
@@ -283,4 +372,6 @@ module.exports = {
   reset,
   seq,
   Sequelize,
+  getDBReplicationObject,
+  getReadOnlyDBConfig,
 };

tests/config/configUtil.js

@@ -109,3 +109,35 @@ describe('csvToArray', () => {
     done();
   });
 }); // csvToArray
+
+describe('csvToArray', () => {
+  it('undefined string', (done) => {
+    expect(configUtil.csvToArray(undefined))
+    .to.be.eql([]);
+    done();
+  });
+
+  it('null string', (done) => {
+    expect(configUtil.csvToArray(null))
+    .to.be.eql([]);
+    done();
+  });
+
+  it('zero-length string', (done) => {
+    expect(configUtil.csvToArray(''))
+    .to.be.eql([]);
+    done();
+  });
+
+  it('single element', (done) => {
+    expect(configUtil.csvToArray('abc'))
+    .to.be.eql(['abc']);
+    done();
+  });
+
+  it('multiple elements with extra left and right padding', (done) => {
+    expect(configUtil.csvToArray('abc,def , ghi'))
+    .to.be.eql(['abc', 'def', 'ghi']);
+    done();
+  });
+}); // csvToArray

tests/db/utils.js

@@ -13,6 +13,7 @@ const expect = require('chai').expect;
 const u = require('../../db/utils');
 
 describe('db utils', () => {
+
   it('dbConfigObjectFromDbURL localhost', (done) => {
     const dbconfig = u.dbConfigObjectFromDbURL(
       'postgres://postgres:postgres@localhost:5432/focusdb');
@@ -35,4 +36,75 @@ describe('db utils', () => {
     expect(dbconfig).to.have.property('port', '5432');
     done();
   }); // dbConfigObjectFromDbURL heroku-ish
+
+  it('getReadOnlyDBConfig from localurl', (done) => {
+    const replicas = ['postgres://postgres:postgres@localhost:5432/focusdb',
+    'postgres://postgres:postgres@localhost:9432/focusdb'
+    ];
+    const replicaConfig = u.getReadOnlyDBConfig(replicas);
+    expect(replicaConfig).to.have.lengthOf(2);
+    expect(replicaConfig[0]).to.have.property('username', 'postgres');
+    expect(replicaConfig[0]).to.have.property('password', 'postgres');
+    expect(replicaConfig[0]).to.have.property('host', 'localhost');
+    expect(replicaConfig[0]).to.have.property('port', '5432');
+
+    expect(replicaConfig[1]).to.have.property('username', 'postgres');
+    expect(replicaConfig[1]).to.have.property('password', 'postgres');
+    expect(replicaConfig[1]).to.have.property('host', 'localhost');
+    expect(replicaConfig[1]).to.have.property('port', '9432');
+
+
+    done();
+  }); // getReadOnlyDBConfig local
+
+  it('getReadOnlyDBConfig from heroku-ish url', (done) => {
+    const replicas =
+    ['postgres://user1:pwd1@ec2-00-00-000-000.compute-0.amazonaws.com:5432/db',
+    'postgres://user1:pwd1@ec2-00-00-000-111.compute-0.amazonaws.com:5432/db'
+    ];
+    const replicaConfig = u.getReadOnlyDBConfig(replicas);
+    expect(replicaConfig).to.have.lengthOf(2);
+    expect(replicaConfig[0]).to.have.property('username', 'user1');
+    expect(replicaConfig[0]).to.have.property('password', 'pwd1');
+    expect(replicaConfig[0]).to.have.property('host',
+      'ec2-00-00-000-000.compute-0.amazonaws.com');
+    expect(replicaConfig[0]).to.have.property('port', '5432');
+
+    expect(replicaConfig[1]).to.have.property('username', 'user1');
+    expect(replicaConfig[1]).to.have.property('password', 'pwd1');
+    expect(replicaConfig[1]).to.have.property('host',
+      'ec2-00-00-000-111.compute-0.amazonaws.com');
+    expect(replicaConfig[1]).to.have.property('port', '5432');
+
+
+    done();
+  }); // getReadOnlyDBConfig heroku-ish
+
+  it('getDBReplicationObject from local', (done) => {
+    const dbconfig = u.dbConfigObjectFromDbURL(
+    'postgres://postgres:postgres@localhost:5432/focusdb');
+    const seqRepConf = u.getDBReplicationObject(dbconfig);
+    expect(seqRepConf).to.have.property('write');
+    expect(seqRepConf.read).to.equal(undefined);
+    expect(seqRepConf.write).to.have.property('username', 'postgres');
+    expect(seqRepConf.write).to.have.property('password', 'postgres');
+    expect(seqRepConf.write).to.have.property('host', 'localhost');
+    expect(seqRepConf.write).to.have.property('port', '5432');
+    done();
+  }); // getDBReplicationObject local
+
+  it('getDBReplicationObject from heroku-ish url', (done) => {
+    const dbconfig = u.dbConfigObjectFromDbURL(
+    'postgres://user1:pwd1@ec2-00-00-000-000.compute-0.amazonaws.com:5432/db');
+    const seqRepConf = u.getDBReplicationObject(dbconfig);
+    expect(seqRepConf).to.have.property('write');
+    expect(seqRepConf.read).to.equal(undefined);
+    expect(seqRepConf.write).to.have.property('username', 'user1');
+    expect(seqRepConf.write).to.have.property('password', 'pwd1');
+    expect(seqRepConf.write).to.have.property('host',
+      'ec2-00-00-000-000.compute-0.amazonaws.com');
+    expect(seqRepConf.write).to.have.property('port', '5432');
+    done();
+  }); // getDBReplicationObject heroku-ish
+
 }); // db utils