Move DB from MongoDB to PostgreSQL

For a number of reasons, we've decided that MongoDB is unsuitable for this project. The nature of our data is relational, and getting everything to work together will require some hacks and be brittle in nature. Reasons for the switch to Postgres include:

* To store emails and associate them with users we have two options. Either we embed emails in user documents and use GridFs when they exceed 16mb, or we place them all in a single collection and search them via an index. The first option results in the inability to run queries for particular emails and has a number of other performance issue. The second method will result in potentially billions of emails in a single document, with queries suffering in performance against such a large number.
* MongoDB is not ACID compliant. Atomicitiy and isolation are particularly important as they ensure predictability when uploading large datasets and allay concurrency concerns with multi-user accounts.
* In light of the above, Postgres is ACID compliant & relational. It also offers additional features useful for handling multi-user accounts & permissions that will be useful in protecting sensitive data (emails).

.sequelizerc

@@ -0,0 +1,6 @@
+module.exports = {
+  config: './server/config/sequelize_config.js',
+  migrationsPath: './server/migrations',
+  modelsPath: './server/models',
+  seedersPath: './server/seed'
+};

package.json

@@ -20,7 +20,6 @@
     "client:test:cover:travis": "babel-node node_modules/isparta/bin/isparta cover --root client--report lcovonly _mocha -- --require ./tools/testSetup.js \"client/**/*.spec.js\" && cat ./coverage/lcov.info | node_modules/coveralls/bin/coveralls.js",
     "client:test:watch": "npm run client:test -- --watch",
     "client:open:cover": "npm run test:cover && open coverage/index.html",
-
     "start": "node server/index.js",
     "test": "echo \"Error: no test specified\" && exit 1",
     "build": "babel-node tools/buildServer.js",
@@ -50,21 +49,23 @@
     "jQuery": "^1.7.4",
     "jquery": "^3.1.0",
     "lodash": "^4.15.0",
-    "mongoose": "^4.6.0",
     "nodemailer": "^2.6.0",
     "nodemailer-ses-transport": "^1.4.0",
     "object-assign": "4.1.0",
     "papaparse": "^4.1.2",
     "passport": "^0.3.2",
     "passport-google-oauth": "^1.0.0",
+    "pg": "^6.1.0",
+    "pg-hstore": "^2.3.2",
     "react": "15.3.1",
     "react-bootstrap": "^0.30.3",
     "react-dom": "15.3.1",
     "react-redux": "4.4.5",
     "react-router": "2.7.0",
     "react-router-redux": "4.0.5",
     "redux": "3.5.2",
-    "redux-thunk": "2.1.0"
+    "redux-thunk": "2.1.0",
+    "sequelize": "^3.24.3"
   },
   "devDependencies": {
     "autoprefixer": "6.4.0",
@@ -115,6 +116,7 @@
     "replace": "0.3.0",
     "rimraf": "2.5.4",
     "sass-loader": "4.0.0",
+    "sequelize-cli": "^2.4.0",
     "sinon": "1.17.5",
     "sinon-chai": "2.8.0",
     "smtp-server": "^1.14.2",

server/config/passport.js

@@ -1,20 +1,28 @@
-const Google = require('./passport/google');
-
-const User = require('../db/models').User;
 const secrets = require('./secrets');
+const User = require('../models').User;
 
 module.exports = (passport) => {
+    const db = require('../models');
+    const Google = require('./passport/google');
+
+    const sequelize = db.sequelize;
+
     passport.serializeUser(function(user,done) {
         done(null, user.id);
     });
 
     passport.deserializeUser(function(id, done) {
-        User.findById(id, (err, user) => {
-            done(err, user);
-        });
+
+        User.findById(id)
+        .then(user => {
+            done(null, user);
+        })
+        .catch(err => {
+            if (err) throw err;
+        })
     });
     ///////////////////////////////
     /* AUTHENTICATION STRATEGIES */
     ///////////////////////////////
-    Google(passport, User, secrets.google);
+    Google(passport, secrets.google);
 };

server/config/passport/google.js

@@ -1,6 +1,7 @@
 const GoogleStrategy = require('passport-google-oauth').OAuth2Strategy;
+const User = require('../../models').User;
 
-module.exports = (passport, User, secret) => {
+module.exports = (passport, secret) => {
 
     passport.use(new GoogleStrategy({
         clientID: secret.consumerKey,
@@ -9,8 +10,43 @@ module.exports = (passport, User, secret) => {
     }, (token, tokenSecret, profile, done) => {
 
         User.findOne({
-            'google.id': profile.id
-        }, (err, userExists) => {
+            where: {
+                googleId: profile.id
+            }
+        }).then(userExists => {
+            if (userExists) {
+                done(null, userExists);
+            } else {
+                User.create({
+                    googleId: profile.id,
+                    token: token,
+                    email: profile._json.emails[0].value,
+                    name: profile.displayName,
+                    picture: profile._json.picture
+                }).then(newUser => {
+                    done(null, newUser);
+                });
+            }
+
+        }).catch(err => {
+            throw err;
+        });
+
+    }))
+}
+
+
+
+/*User.findOne({
+            where: { google: profile.id }
+        }).then(existingUser) => {
+            console.log(exists)
+        }
+
+    }))*/
+
+
+/*(err, userExists) => {
             // Search will proceed in the following manner:
             //  1. If there was an error, return the error
             //  2. If found, serialise user
@@ -41,3 +77,4 @@ module.exports = (passport, User, secret) => {
 
     }));
 };
+*/

server/config/sequelize_config.js

@@ -0,0 +1,23 @@
+module.exports = {
+  "development": {
+    "username": process.env.PSQL_USERNAME,
+    "password": process.env.PSQL_PASSWORD,
+    "database": process.env.PSQL_DATABASE,
+    "host": "127.0.0.1",
+    "dialect": "postgres"
+  },
+  "test": {
+    "username": "root",
+    "password": null,
+    "database": "database_test",
+    "host": "127.0.0.1",
+    "dialect": "mysql"
+  },
+  "production": {
+    "username": "root",
+    "password": null,
+    "database": "database_production",
+    "host": "127.0.0.1",
+    "dialect": "mysql"
+  }
+}

server/controllers/addsubscribers.js

@@ -1,32 +1,32 @@
-'use strict'
-const SubscriberModel = require('../db/models').Subscriber;
+const SubscriberModel = require('../models').subscriber;
 
-module.exports =  function(req, res) {
-  const subscribers = req.body.subscribers;
-  const fields = req.body.fields;
+module.exports = function(req, res) {
+    const subscribers = req.body.subscribers;
+    const fields = req.body.fields;
 
-  console.log("got new subscribers: ");
-  console.log(subscribers);
-  console.log(fields);
-
-  subscribers.forEach(subscriber => {
-
-    SubscriberModel.find({email:subscriber.email}, (err, doc) => {
-      if (err) throw err;
-      if (doc.length) {
-        // User exists! Do not save doc
-        res.send({
-          status: 'error',
-          message:'This email already exists'
-        });
-      } else {
-        let newSubscriber = new SubscriberModel();
-        newSubscriber.email = subscriber.email;
-        newSubscriber.save(err => {
-          if (err) throw err;
-        });
-      }
-    })
-  });
-  // -> use schema + add all the subscribers + field types to the database
+    subscribers.forEach(subscriber => {
+        SubscriberModel.findOne({
+            where: {
+                email:subscriber.email
+            }
+        }).then(email => {
+            if (email) {
+                res.status(400)
+                    .send({
+                  status: 'error', // Redundant
+                  message:'This email already exists'
+                });
+            } else {
+                SubscriberModel.create({
+                    email: subscriber.email
+                }).then(() => {
+                    res.status(201)
+                        .send({
+                      status: 'success', // Redundant
+                      message: `${subscriber.email} added succesfully`
+                    });
+                });
+            }
+        })
+    });
 }

server/controllers/changesettings.js

@@ -1,36 +1,43 @@
 'use strict'
 const _ = require('lodash');
-const Settings = require('../db/models/index').Settings;
+const Settings = require('../models').settings;
 
 
 module.exports =  function(req, res) {
   const settingsToChange = _.pickBy(req.body);
 
   // Exit if there are no settings to change
   if (_.isEmpty(settingsToChange)) {
-    res.send({
-      type: 'error',
+    res.status(400)
+        .send({
+      type: 'error', // Redundant
       message: 'SES credentials form was empty'
     });
     return;
   }
 
-  // Should eventually refactor this to use findOneAndSave
-  Settings.findOne({}, {}, (err, settings) => {
-    if (err) throw err;
+  /*
+    Previous model needs to be refactored. Code left below for reference.
+  */
+}
 
-    // Create default settings if none exist
-    if (!settings) {
-      settings = Settings(settingsToChange);
-    } else {
-      _.keys(settingsToChange).forEach((setting) => {
-        settings[setting] = settingsToChange[setting]
-      });
-    }
+/*
+// Should eventually refactor this to use findOneAndSave
+Settings.findOne({}, {}, (err, settings) => {
+  if (err) throw err;
 
-    settings.save((err) => {
-      if (err) throw err;
-      res.json({})
+  // Create default settings if none exist
+  if (!settings) {
+    settings = Settings(settingsToChange);
+  } else {
+    _.keys(settingsToChange).forEach((setting) => {
+      settings[setting] = settingsToChange[setting]
     });
-  })
-}
+  }
+
+  settings.save((err) => {
+    if (err) throw err;
+    res.json({})
+  });
+})
+*/

server/index.js

@@ -1,7 +1,7 @@
 const express = require('express');
 const session = require('express-session')
 const passport = require('passport');
-const mongoose = require('mongoose');
+const Sequelize = require('sequelize');
 const bodyParser = require('body-parser');
 const path = require('path');
 
@@ -12,9 +12,6 @@ const routes = require('./routes');
 
 const app = express();
 
-// Connect to mongo
-mongoose.connect(secret.mongo);
-
 // Config
 require('./config/passport')(passport);