Permalink
Browse files

Add apiVersion to storage and passport auth

Fix Dropbox authentication by allowing the specification of API version 2 (as a string) now that they've completely disabled version 1.

Also:

- Move server file to index.js per standard
- Convert most environment variables into optional ones with defaults
- Remove remote repopulate collections script
- Update README with removal of deployment scripts in favor of grunt-hoist
- Remove port and host attributes from app object for better separation of concerns
- Update dependencies
  • Loading branch information...
markmhx committed Nov 5, 2017
1 parent fcbe0b8 commit d9b1f15400201ef962a8dea79a121ad9d996c686
View
@@ -3,7 +3,7 @@
* @module
*/
var ranger = require('park-ranger')();
require('park-ranger')();
var loadGruntTasks = require('load-grunt-tasks');
module.exports = function(grunt) {
@@ -43,7 +43,7 @@ module.exports = function(grunt) {
},
nodemon: {
dev: {
script: 'app/server.js'
script: 'index.js'
}
},
symlink: {
@@ -56,17 +56,6 @@ module.exports = function(grunt) {
}]
}
},
sshexec: {
options: {
agent: ranger.env.SSH_AUTH_SOCK,
host: ranger.env.SYNC_SERVER_DEPLOY_HOST,
username: ranger.env.SYNC_SERVER_DEPLOY_HOST_USER,
port: 22
},
repopulateCollections: {
command: 'cd ' + ranger.env.SYNC_SERVER_DEPLOY_HOST_DIR + ' && grunt repopulate-collections'
}
},
watch: {
jsdoc: {
files: [
@@ -99,10 +88,6 @@ module.exports = function(grunt) {
'jsdoc:build'
]);
grunt.registerTask('remote-repopulate-collections', 'Remove database collections and repopulate them with resourceObjects stored in files.', [
'sshexec:repopulateCollections'
]);
grunt.registerTask('test', 'Run tests against app.', [
'env:test',
'eslint',
View
@@ -1,4 +1,4 @@
# sync-server
# sync-server 📡💾🏠
[![Codeship badge](https://codeship.com/projects/00364600-b0b2-0133-c9a4-72e14e42ee1c/status?branch=master)](https://app.codeship.com/projects/132772)
[![Code Climate](https://codeclimate.com/github/neotoma/sync-server/badges/gpa.svg)](https://codeclimate.com/github/neotoma/sync-server)
@@ -7,47 +7,43 @@
This repository contains the source code for an app that synchronizes data from sources to storage on behalf of users.
See also documentation for [the JavaScript source code](http://neotoma.github.io/sync-server/) or [the Neotoma project](https://github.com/neotoma/documentation).
See also [the API documentation](http://neotoma.github.io/sync-server/) or [the Neotoma project in general](https://github.com/neotoma/documentation).
## Setup
SSL certificates and the following environment variables are managed by [Park Ranger](https://github.com/markmhx/park-ranger):
- `SYNC_SERVER_DIR`: Local path to app base directory (e.g. `/var/www/sync-server`; required to run tasks)
- `SYNC_SERVER_HOST`: Host address for the app (e.g. `127.0.0.1`; required to run app)
- `SYNC_SERVER_HTTPS_PORT`: Port through which to run the app with HTTPS (e.g. `1234`; required to run app)
- `SYNC_SERVER_HTTP_PORT`: Port through which to run the app with HTTP (e.g. `1235`; required to run app)
- `SYNC_SERVER_SESSION_SECRET`: Secret, non-obvious string used to prevent session tampering (e.g. `oc]7kwM)R*UX3&` but *generate your own*; required to run app)
- `SYNC_SERVER_DEPLOY_HOST_USERNAME`: User name with which to SSH into remote deployment server (e.g. `root`; required to deploy app)
- `SYNC_SERVER_WEB_HOST`: Host address for the web client intended to communicate with the app exclusively via cross-origin HTTP requests; used to set HTTP access control (CORS) (e.g. `http://example.com:9019`; required)
- `SYNC_SERVER_DEPLOY_CERTS_DIR`: Local system path to a directory with the SSL certificate files `key`, `crt` and `ca` needed by the app to serve HTTPs requests *remotely on the deployment server* (e.g. `/var/www/sync-server/.cert-deploy`; required to deploy app). This directory will be copied to `.cert` within the base directory of the app on the deployment server so the environment variable `SYNC_SERVER_CERTS_DIR` must be set to `.cert` in the deployment environment unless this directory is later moved.
- `SYNC_SERVER_DEPLOY_HOST`: Host address for the remote deployment server (e.g. `example.com`; required to deploy app)
- `SYNC_SERVER_DEPLOY_HOST_DIR`: Remote system path to app directory on deployment server (e.g. `/var/www/sync-server`; required to deploy app)
- `SYNC_SERVER_DEPLOY_SYSTEMD_SERVICE`: Name of systemd service on host that should be restarted or started upon app deployment (e.g. `syncserver`; optional to deploy app)
- `SSH_AUTH_SOCK`: Socket used by system for SSH agent forwarding (required to deploy app)
- `SYNC_SERVER_MONGODB_DATABASE`: Name of a [MongoDB](http://www.mongodb.org/) database (e.g. `sync_server`)
- `SYNC_SERVER_MONGODB_HOST`: Host address for a MongoDB service (e.g. `127.0.0.1`; required to run app)
- `SYNC_SERVER_MONGODB_PORT`: Port through which to access a MongoDB service (e.g. `27017`; required to run app)
- `SYNC_SERVER_MAILER_LOGGER_EMAIL`: Email to which logger errors should be sent (optional to run app)
- `SYNC_SERVER_LOGGER_MAILER_LEVEL`: Numeric value between 0 and 5 to designate level of errors to email to SYNC_SERVER_MAILER_LOGGER_EMAIL (optional to run app)
- `SYNC_SERVER_NAME`: Name used by the app to identity itself with users (e.g. `Neotoma`; required to run app)
- `SYNC_SERVER_MAILER_SENDER_EMAIL`: Email address used by app to send email (e.g. `support@example.com`; required to run app)
- `SYNC_SERVER_MAILER_DEV_RECIPIENT_EMAIL`: Email address used by the app to manually test the delivery of email (e.g. `developer@example.com`; required to run app in the development environment but not required to run it in other environments nor to deploy)
- `SYNC_SERVER_MAILER_LOGGER_EMAIL`: Email address used by the app to report high-priority log events by email (e.g. `developer-support@example.com`; optional)
- `SYNC_SERVER_LOGGER_FILE_PATH`: File system path where to store log events (e.g. `/tmp/sync-server.log`; optional)
- `SYNC_SERVER_CERTS_DIR`: Local system path to a directory with the SSL certificate files `key`, `crt` and `ca` needed by the app to serve HTTPs requests (e.g. `/var/www/sync-server/.cert`; required to run app)
- `SYNC_SERVER_SENDGRID_API_KEY`: API key for SendGrid account for delivering email (optional to run app)
- `SYNC_SERVER_MAILER_RECIPIENT_EMAIL`: Email address to which to restrict all email delivery for testing purposes (optional to run app)
### Required
- `SYNC_SERVER_SESSION_SECRET`: Secret, non-obvious string used to prevent session tampering (e.g. `oc]7kwM)R*UX3&` but *generate your own*)
### Optional
- `SYNC_SERVER_DIR`: Local path to app base directory (defaults to `/var/www/sync-server`)
- `SYNC_SERVER_HOST`: Host address for the app (defaults to `127.0.0.1`)
- `SYNC_SERVER_HTTP_PORT`: Port through which to run the app with HTTP (defaults `9001`)
- `SYNC_SERVER_HTTPS_PORT`: Port through which to run the app with HTTPS (defaults to `9002`)
- `SYNC_SERVER_LOGGER_FILE_PATH`: File system path where to store log events (e.g. `/tmp/sync-server.log`)
- `SYNC_SERVER_LOGGER_MAILER_LEVEL`: Numeric value between 0 and 5 to designate level of errors to email to SYNC_SERVER_MAILER_LOGGER_EMAIL
- `SYNC_SERVER_MAILER_LOGGER_EMAIL`: Email to which logger errors should be sent
- `SYNC_SERVER_MAILER_RECIPIENT_EMAIL`: Email address to which to restrict all email delivery for testing purposes
- `SYNC_SERVER_MAILER_SENDER_EMAIL`: Email address used by app to send email (e.g. `support@example.com`; required to send email)
- `SYNC_SERVER_MONGODB_DATABASE`: Name of a [MongoDB](http://www.mongodb.org/) database (defaults to `sync_server`)
- `SYNC_SERVER_MONGODB_HOST`: Host address for a MongoDB service (defaults to `127.0.0.1`)
- `SYNC_SERVER_MONGODB_PORT`: Port through which to access a MongoDB service (defaults to `27017`)
- `SYNC_SERVER_NAME`: Name used by the app to identity itself with users (defaults to `Neotoma`)
- `SYNC_SERVER_SENDGRID_API_KEY`: API key for SendGrid account for delivering email (required to send email)
- `SYNC_SERVER_WEB_HOST`: Host address for the web client intended to communicate with the app exclusively via cross-origin HTTP requests; used to set HTTP access control (CORS) (e.g. `http://example.com:9019`)
Tests will use Park Ranger to establish environment variables available for the "test" environment after loading those for whatever environment initially indicated upon execution.
Be sure to set any of the above variables to a different value within `.env-test` if you don't want the tests to use the variables available for the indicated environment.
For example, set a different `SYNC_SERVER_MONGODB_DATABASE` to prevent your development database from getting reset every time you run tests, and `SYNC_SERVER_SENDGRID_API_KEY` to "null" to prevent email delivery.
For example, set a different `SYNC_SERVER_MONGODB_DATABASE` to prevent your development database from getting reset every time you run tests, and `SYNC_SERVER_SENDGRID_API_KEY` to `null` to prevent email delivery.
## Running the server
Once the environment is ready per above, and [Node.js](http://nodejs.org/) with [NPM](https://www.npmjs.com/) is installed, simply run `npm install` to install dependencies in the `node_modules` directory and `node server.js` to fire the server up.
Once the environment is ready per above, and [Node.js](http://nodejs.org/) with [NPM](https://www.npmjs.com/) is installed, simply run `npm install` to install dependencies in the `node_modules` directory and `npm start` to fire the server up.
## Developing and deploying the server
View
@@ -3,18 +3,10 @@
* @module
*/
if (!process.env.SYNC_SERVER_MONGODB_DATABASE) {
throw new Error('MongoDB not provided database name');
}
if (!process.env.SYNC_SERVER_MONGODB_HOST) {
throw new Error('MongoDB failed to find host variable environment');
}
if (!process.env.SYNC_SERVER_MONGODB_PORT) {
throw new Error('MongoDB port not provided by environment');
}
var database = process.env.SYNC_SERVER_MONGODB_DATABASE ? process.env.SYNC_SERVER_MONGODB_DATABASE : 'sync_server';
var host = process.env.SYNC_SERVER_MONGODB_HOST ? process.env.SYNC_SERVER_MONGODB_HOST : '127.0.0.1';
var port = process.env.SYNC_SERVER_MONGODB_PORT ? process.env.SYNC_SERVER_MONGODB_PORT : 27017;
module.exports = {
url: 'mongodb://' + process.env.SYNC_SERVER_MONGODB_HOST + ':' + process.env.SYNC_SERVER_MONGODB_PORT + '/' + process.env.SYNC_SERVER_MONGODB_DATABASE
url: `mongodb://${host}:${port}/${database}`
};
View
@@ -27,16 +27,6 @@ app.use(expressSession({
app.use(passport.initialize());
app.use(passport.session());
if (!process.env.SYNC_SERVER_HOST) {
throw new Error('App failed to find host variable from environment');
}
if (!process.env.SYNC_SERVER_HTTPS_PORT) {
throw new Error('App failed to find HTTPS port variable from environment');
}
app.host = 'https://' + process.env.SYNC_SERVER_HOST + ':' + process.env.SYNC_SERVER_HTTPS_PORT;
app.requireAdminAuthentication = function(req, res, next) {
if (!req.user || !req.user.id || !req.user.admin) {
res.status(403).send('403 Forbidden');
View
@@ -5,20 +5,20 @@ var sendGridTransport = require('nodemailer-sendgrid-transport');
var stubTransport = require('nodemailer-stub-transport');
var mailer = {
emailRegex: /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/
emailRegex: /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ // eslint-disable-line
};
mailer.sendMail = function(email, done) {
try {
if (!process.env.SYNC_SERVER_NAME) {
throw new Error('Mailer failed to find app name variable in environment');
}
var senderEmail = process.env.SYNC_SERVER_MAILER_SENDER_EMAIL;
var serverName = process.env.SYNC_SERVER_NAME ? process.env.SYNC_SERVER_NAME : 'Neotoma';
if (!process.env.SYNC_SERVER_MAILER_SENDER_EMAIL) {
throw new Error('Mailer failed to find sender email variable in environment');
if (!senderEmail) {
debug('Sending mail aborted due to lack of SYNC_SERVER_MAILER_SENDER_EMAIL environment variable');
return done();
}
email.from = process.env.SYNC_SERVER_NAME + '<' + process.env.SYNC_SERVER_MAILER_SENDER_EMAIL + '>';
email.from = `${serverName}<${senderEmail}>`;
if (logger.trace) {
logger.trace('Mailer sending email', { email: email });
View
@@ -16,7 +16,9 @@ mongoose.connection.once('open', function() {
logger.info('Mongoose connected to MongoDB', { url: mongoDBConfig.url });
});
mongoose.connect(mongoDBConfig.url);
mongoose.connect(mongoDBConfig.url, {
useMongoClient: true
});
mongoose.transform = function(doc, ret) {
delete ret._id;
@@ -22,7 +22,7 @@ module.exports = function(grunt) {
var models = require('app/models');
var mongoose = require('app/lib/mongoose');
var dataPath = path.resolve(process.env.SYNC_SERVER_DIR, 'data');
var dataPath = path.resolve(process.env.SYNC_SERVER_DIR ? process.env.SYNC_SERVER_DIR : '/var/www/sync-server', 'data');
debug.start('# repopulating collections: %s', dataPath);
View
@@ -1,4 +1,4 @@
/**
* Regex to validate URLs
*/
module.exports = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%_\+.~#?&//=]{2,256}\.[a-z]{2,4}\b(\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?/;
module.exports = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%_\+.~#?&//=]{2,256}\.[a-z]{2,4}\b(\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?/; // eslint-disable-line
@@ -1,4 +1,4 @@
/**
/**
* NotificationRequest model
* @module
*/
View
@@ -20,6 +20,7 @@ var validateParams = require('app/lib/validateParams');
* @property {string} [itemPutUrlTemplate=https://{$host}{$path}?access_token={$accessToken}] - String template used to generate URLs for PUT requests for items to storage
*/
module.exports = modelFactory.new('Storage', {
apiVersion: String,
clientId: String,
clientSecret: String,
host: String,
View
@@ -66,11 +66,12 @@ var reqPassportDocument = function(req, res, next) {
var passportStrategy = require(document.passportStrategy);
req.strategy = new passportStrategy.Strategy({
apiVersion: document.apiVersion,
clientID: document.clientId,
clientSecret: document.clientSecret,
consumerKey: document.clientId,
consumerSecret: document.clientSecret,
callbackURL: app.host + path.resolve('/', Model.modelType(), document.slug, 'auth-callback'),
callbackURL: `${req.protocol}://${req.get('host')}${path.resolve('/', Model.modelType(), document.slug, 'auth-callback')}`,
passReqToCallback: true,
profileFields: ['id', 'displayName', 'emails']
}, function(req, accessToken, refreshToken, profile, done) {
View
@@ -338,6 +338,7 @@ module.exports = {
mockProperties: () => {
return {
_id: ObjectId(),
apiVersion: '5',
clientId: 'storageClientId',
clientSecret: 'storageClientSecret',
host: 'storagehost.example.com',
@@ -349,6 +350,7 @@ module.exports = {
};
},
schemaProperties: {
apiVersion: String,
clientId: String,
clientSecret: String,
host: String,
View
@@ -12,12 +12,15 @@ var passportSocketIO = require('app/lib/passportSocketIO');
var socketEvents = require('app/socketEvents');
var socketIO = require('socket.io');
var httpsServer = https.createServer(ranger.cert, app).listen(process.env.SYNC_SERVER_HTTPS_PORT, () => {
debug('App server started listening for HTTPS requests', { port: process.env.SYNC_SERVER_HTTPS_PORT });
var httpPort = process.env.SYNC_SERVER_HTTP_PORT ? process.env.SYNC_SERVER_HTTP_PORT : 9001;
var httpsPort = process.env.SYNC_SERVER_HTTPS_PORT ? process.env.SYNC_SERVER_HTTPS_PORT : 9002;
var httpServer = http.createServer(app).listen(httpPort, () => {
debug('App server started listening for HTTP requests', { port: httpPort });
});
var httpServer = http.createServer(app).listen(process.env.SYNC_SERVER_HTTP_PORT, () => {
debug('App server started listening for HTTP requests', { port: process.env.SYNC_SERVER_HTTP_PORT });
var httpsServer = https.createServer(ranger.cert, app).listen(httpsPort, () => {
debug('App server started listening for HTTPS requests', { port: httpsPort });
});
var servers = {
View
@@ -5,24 +5,24 @@
"body-parser": "^1.12.0",
"colors": "^1.1.2",
"compression": "^1.4.1",
"connect-mongo": "^1.3.2",
"connect-mongo": "^2.0.0",
"cookie-parser": "^1.4.3",
"cors": "^2.8.1",
"debug": "^2.6.0",
"debug": "^3.1.0",
"emoji-strip": "^1.0.0",
"es6-template-strings": "^2.0.1",
"eslint": "^3.17.1",
"express": "~4.15.0",
"eslint": "^4.10.0",
"express": "~4.16.2",
"express-session": "^1.14.1",
"flat": "^2.0.1",
"flat": "^4.0.0",
"format-json": "^1.0.3",
"fs": "0.0.2",
"grunt": "~1.0.1",
"grunt-cli": "^1.2.0",
"grunt-contrib-clean": "^1.0.0",
"grunt-contrib-symlink": "^1.0.0",
"grunt-contrib-watch": "^1.0.0",
"grunt-eslint": "^19.0.0",
"grunt-eslint": "^20.1.0",
"grunt-force-task": "^2.0.0",
"grunt-jsdoc": "^2.1.0",
"grunt-mocha-test": "^0.13.2",
@@ -36,36 +36,36 @@
"mime-types": "^2.1.12",
"minami": "^1.1.1",
"minimist": "^1.2.0",
"mocha": "^3.2.0",
"mocha": "^4.0.1",
"mongodb": "^2.2.24",
"mongoose": "^4.7.5",
"mongoose-autopopulate": "^0.5.0",
"mongoose-autopopulate": "^0.6.0",
"morgan": "^1.7.0",
"nock": "^9.0.9",
"nodemailer": "^2.6.4",
"nodemailer": "^4.3.1",
"nodemailer-sendgrid-transport": "^0.2.0",
"nodemailer-stub-transport": "^1.1.0",
"park-ranger": "1.0.11",
"passport": "~0.3.2",
"passport": "~0.4.0",
"passport-dropbox-oauth2": "~1.1.0",
"passport-facebook": "^2.1.1",
"passport-foursquare": "~1.0.0",
"passport-instagram": "^1.0.0",
"passport-mocked": "^1.0.1",
"passport-twitter": "^1.0.4",
"passport.socketio": "^3.4.1",
"pluralize": "^4.0.0",
"pluralize": "~7.0.0",
"request": "^2.75.0",
"require-dir": "^0.3.1",
"rsync": "^0.6.1",
"sanitize-filename": "^1.6.1",
"sinon": "^1.17.6",
"socket.io": "^1.3.2",
"sinon": "~4.1.0",
"socket.io": "~2.0.4",
"supertest": "^3.0.0",
"underscore": "^1.8.3",
"url": "^0.11.0",
"winston": "^2.3.0",
"winston-mongodb": "^2.0.0"
"winston-mongodb": "~3.0.1"
},
"license": "MIT",
"name": "sync-server",
@@ -76,6 +76,7 @@
"rebuild-jsdoc": "grunt rebuild-jsdoc",
"postinstall": "grunt symlink:modules",
"repopulate-collections": "grunt repopulate-collections",
"start": "node index.js",
"test": "grunt test",
"deploy-all": "grunt hoist-deploy-all",
"deploy-app": "grunt hoist-deploy-app",
View
@@ -15,8 +15,4 @@ describe('app', function() {
it('has requireAthentication method', function() {
assert(typeof app.requireAuthentication === 'function');
});
it('has host', function() {
assert(typeof app.host === 'string');
});
});

0 comments on commit d9b1f15

Please sign in to comment.