Moved knex-migrator execution into Ghost

refs TryGhost#9742, refs TryGhost/Ghost-CLI#759

- required a reordering of Ghost's bootstrap file, because:
  - we have to ensure that no database queries are executed within Ghost during the migrations
  - make 3 sections: check if db needs initialisation, bootstrap Ghost with minimal components (db/models, express apps, load settings+theme)
- create a new `migrator` utility, which tells you which state your db is in and offers an API to execute knex-migrator based on this state
- ensure we still detect an incompatible db: you connect your 2.0 blog with a 0.11 database
- enable maintenance mode if migrations are missing
- if the migration have failed, knex-migrator roll auto rollback
  - you can automatically switch to 1.0 again
- added socket communication for the CLI

core/server/adapters/scheduling/SchedulingDefault.js

@@ -234,6 +234,14 @@ SchedulingDefault.prototype._pingUrl = function (object) {
                     object.tries = tries + 1;
                     self._pingUrl(object);
                 }, self.retryTimeoutInMs);
+
+                common.logging.error(new common.errors.GhostError({
+                    err: err,
+                    context: 'Retrying...',
+                    level: 'normal'
+                }));
+
+                return;
             }
 
             common.logging.error(new common.errors.GhostError({

core/server/data/db/migrator.js

@@ -0,0 +1,72 @@
+const KnexMigrator = require('knex-migrator'),
+    config = require('../../config'),
+    common = require('../../lib/common'),
+    knexMigrator = new KnexMigrator({
+        knexMigratorFilePath: config.get('paths:appRoot')
+    });
+
+module.exports.getState = () => {
+    let state, err;
+
+    return knexMigrator.isDatabaseOK()
+        .then(() => {
+            state = 1;
+            return state;
+        })
+        .catch((_err) => {
+            err = _err;
+
+            // CASE: database was never created
+            if (err.code === 'DB_NOT_INITIALISED') {
+                state = 2;
+                return state;
+            }
+
+            // CASE: you have created the database on your own, you have an existing none compatible db?
+            if (err.code === 'MIGRATION_TABLE_IS_MISSING') {
+                state = 3;
+                return state;
+            }
+
+            // CASE: database needs migrations
+            if (err.code === 'DB_NEEDS_MIGRATION') {
+                state = 4;
+                return state;
+            }
+
+            // CASE: database connection errors, unknown cases
+            throw err;
+        });
+};
+
+module.exports.dbInit = () => {
+    return knexMigrator.init();
+};
+
+module.exports.migrate = () => {
+    return knexMigrator.migrate();
+};
+
+module.exports.isDbCompatible = (connection) => {
+    return connection.raw('SELECT `key` FROM settings WHERE `key`="databaseVersion";')
+        .then((response) => {
+            if (!response || !response[0].length) {
+                return;
+            }
+
+            throw new common.errors.DatabaseVersionError({
+                message: 'Your database version is not compatible with Ghost 2.0.',
+                help: 'Want to keep your DB? Use Ghost < 1.0.0 or the "0.11" branch.' +
+                      '\n\n\n' +
+                      'Want to migrate Ghost 0.11 to 2.0? Please visit https://docs.ghost.org/v1/docs/migrating-to-ghost-1-0-0'
+            });
+        })
+        .catch((err) => {
+            // CASE settings table doesn't exists
+            if (err.errno === 1146 || err.errno === 1) {
+                return;
+            }
+
+            throw err;
+        });
+};

core/server/data/migrations/hooks/init/shutdown.js

@@ -1,27 +1,20 @@
-var _ = require('lodash'),
-    config = require('../../../../config'),
+const _ = require('lodash'),
     database = require('../../../db');
 
-module.exports = function after() {
-    // do not close database connection in test mode, because all tests are executed one after another
-    // this check is not nice, but there is only one other solution i can think of:
-    // forward a custom object to knex-migrator, which get's forwarded to the hooks
-    if (config.get('env').match(/testing/g)) {
-        return;
-    }
-
-    // running knex-migrator migrate --init does two different migration calls within a single process
-    // we have to ensure that we clear the Ghost cache afterwards, otherwise we operate on a destroyed connection
-    _.each(require.cache, function (val, key) {
-        if (key.match(/core\/server/)) {
-            delete require.cache[key];
-        }
-    });
+module.exports = function shutdown(options = {}) {
+    if (options.executedFromShell === true) {
+        // running knex-migrator migrate --init in the shell does two different migration calls within a single process
+        // we have to ensure that we clear the Ghost cache afterwards, otherwise we operate on a destroyed connection
+        _.each(require.cache, function (val, key) {
+            if (key.match(/core\/server/)) {
+                delete require.cache[key];
+            }
+        });
 
-    // we need to close the database connection
-    // the after hook signals the last step of a knex-migrator command
-    // Example:
-    // Ghost-CLI calls knexMigrator.init and afterwards it starts Ghost, but Ghost-CLI can't shutdown
-    // if Ghost keeps a connection alive
-    return database.knex.destroy();
+        /**
+         * We have to close Ghost's db connection if knex-migrator was used in the shell.
+         * Otherwise the process doesn't exit.
+         */
+        return database.knex.destroy();
+    }
 };

core/server/data/migrations/hooks/migrate/shutdown.js

@@ -1,18 +1,11 @@
-var config = require('../../../../config'),
-    database = require('../../../db');
+const database = require('../../../db');
 
-module.exports = function after() {
-    // do not close database connection in test mode, because all tests are executed one after another
-    // this check is not nice, but there is only one other solution i can think of:
-    // forward a custom object to knex-migrator, which get's forwarded to the hooks
-    if (config.get('env').match(/testing/g)) {
-        return;
+module.exports = function shutdown(options = {}) {
+    /**
+     * We have to close Ghost's db connection if knex-migrator was used in the shell.
+     * Otherwise the process doesn't exit.
+     */
+    if (options.executedFromShell === true) {
+        return database.knex.destroy();
     }
-
-    // we need to close the database connection
-    // the after hook signals the last step of a knex-migrator command
-    // Example:
-    // Ghost-CLI calls knexMigrator.init and afterwards it starts Ghost, but Ghost-CLI can't shutdown
-    // if Ghost keeps a connection alive
-    return database.knex.destroy();
 };

core/server/data/schema/fixtures/utils.js

@@ -116,7 +116,7 @@ addFixturesForModel = function addFixturesForModel(modelFixture, options) {
 
     return Promise.mapSeries(modelFixture.entries, function (entry) {
         // CASE: if id is specified, only query by id
-        return models[modelFixture.name].findOne(entry.id ? {id: entry.id} : entry, options).then(function (found) {
+        return models[modelFixture.name].findOne(entry.id ? {id: entry.id} : entry.slug ? {slug: entry.slug} : entry, options).then(function (found) {
             if (!found) {
                 return models[modelFixture.name].add(entry, options);
             }

core/server/ghost-server.js

@@ -94,7 +94,13 @@ GhostServer.prototype.start = function (externalApp) {
         self.httpServer.on('connection', self.connection.bind(self));
         self.httpServer.on('listening', function () {
             debug('...Started');
-            common.events.emit('server.start');
+
+            // CASE: there are components which listen on this event to initialise after the server has started (in background)
+            //       we want to avoid that they bootstrap during maintenance
+            if (config.get('maintenance:enabled') === false) {
+                common.events.emit('server.start');
+            }
+
             self.logStartMessages();
             resolve(self);
         });
@@ -156,6 +162,8 @@ GhostServer.prototype.hammertime = function () {
 GhostServer.prototype.connection = function (socket) {
     var self = this;
 
+    this.socket = socket;
+
     self.connectionId += 1;
     socket._ghostId = self.connectionId;
 
@@ -166,6 +174,10 @@ GhostServer.prototype.connection = function (socket) {
     self.connections[socket._ghostId] = socket;
 };
 
+GhostServer.prototype.getSocket = function getSocket() {
+    return this.socket;
+};
+
 /**
  * ### Close Connections
  * Most browsers keep a persistent connection open to the server, which prevents the close callback of